From benjamin at codespeak.net Thu Apr 1 01:49:09 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Apr 2010 01:49:09 +0200 (CEST) Subject: [pypy-svn] r73237 - in pypy/branch/cpython-extension/pypy: module/cpyext/include module/cpyext/test rlib Message-ID: <20100331234909.DE4E2282BD8@codespeak.net> Author: benjamin Date: Thu Apr 1 01:49:07 2010 New Revision: 73237 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/intobject.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/patchlevel.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (props changed) pypy/branch/cpython-extension/pypy/rlib/exports.py (props changed) Log: set eol-style on new files From xoraxax at codespeak.net Thu Apr 1 04:23:21 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 04:23:21 +0200 (CEST) Subject: [pypy-svn] r73238 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100401022321.797AC282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 04:23:19 2010 New Revision: 73238 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Introduce dual objects to W_PyCObject. These are used only to be stored in the mapping dicts. Before this change, cpyext would always throw away the pyo, even of a C type, with all its data because the reference count did not consider that PyPy itself was also using the object. Now this is respected by the invariant that the refcount of the object is up by one if the corresponding PyCObject exists. As long as the pyo is alive (refcnt > 0), the PyCObject is automatically recreated if necessary. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Thu Apr 1 04:23:19 2010 @@ -3,7 +3,8 @@ from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject, W_PyCObject +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject, W_PyCObject,\ + W_PyCObjectDual from pypy.objspace.std.objectobject import W_ObjectObject import pypy.module.__builtin__.operation as operation @@ -12,6 +13,11 @@ if isinstance(w_type, W_PyCTypeObject): w_pycobj = space.allocate_instance(W_PyCObject, w_type) w_pycobj.__init__(space) + w_pycobjd = space.allocate_instance(W_PyCObjectDual, w_type) + w_pycobjd.__init__(space) + w_pycobjd.set_pycobject(w_pycobj) + w_pycobj.set_dual(w_pycobjd) + Py_IncRef(space, w_pycobj) return w_pycobj assert False, "Please add more cases in get_cls_for_type_object!" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 1 04:23:19 2010 @@ -29,14 +29,16 @@ def make_ref(space, w_obj, borrowed=False, steal=False): + from pypy.module.cpyext.typeobject import allocate_type_obj,\ + W_PyCTypeObject, W_PyCObject, W_PyCObjectDual if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) state = space.fromcache(State) + if isinstance(w_obj, W_PyCObject): + w_obj = w_obj.w_dual py_obj = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not py_obj: - from pypy.module.cpyext.typeobject import allocate_type_obj,\ - W_PyCTypeObject, W_PyCObject w_type = space.type(w_obj) if space.is_w(w_type, space.w_type): pto = allocate_type_obj(space, w_obj) @@ -97,13 +99,27 @@ def from_ref(space, ref): + from pypy.module.cpyext.typeobject import W_PyCObjectDual, W_PyCObject assert lltype.typeOf(ref) == PyObject if not ref: return None state = space.fromcache(State) ptr = rffi.cast(ADDR, ref) try: - obj = state.py_objects_r2w[ptr] + w_obj = state.py_objects_r2w[ptr] + if isinstance(w_obj, W_PyCObjectDual): + w_obj_wr = w_obj.w_pycobject + w_obj_or_None = w_obj_wr() + if w_obj_or_None is None: + # resurrect new PyCObject + Py_IncRef(space, ref) + w_obj_new = space.allocate_instance(W_PyCObject, space.type(w_obj)) + w_obj_new.__init__(space) + w_obj_new.set_dual(w_obj) + w_obj.set_pycobject(w_obj_new) + w_obj = w_obj_new + else: + w_obj = w_obj_or_None except KeyError: ref_type = ref.c_ob_type if ref != ref_type and space.is_w(from_ref(space, ref_type), space.w_str): @@ -113,7 +129,7 @@ if not we_are_translated(): msg = "Got invalid reference to a PyObject: %r" % (ref, ) raise InvalidPointerException(msg) - return obj + return w_obj # XXX Optimize these functions and put them into macro definitions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Thu Apr 1 04:23:19 2010 @@ -57,11 +57,21 @@ return PyString_FromStringAndSize("Foo Example", 11); } +static PyObject * +foo_get_foo(PyObject *self, void *closure) +{ + return PyInt_FromLong(((fooobject*)self)->foo); +} + static PyGetSetDef foo_getseters[] = { {"name", (getter)foo_get_name, NULL, NULL, NULL}, + {"foo", + (getter)foo_get_foo, NULL, + NULL, + NULL}, {NULL} /* Sentinel */ }; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Thu Apr 1 04:23:19 2010 @@ -10,6 +10,8 @@ assert 'foo' in sys.modules assert "copy" in dir(module.fooType) obj = module.new() + print obj.foo + assert obj.foo == 42 print "Obj has type", type(obj) assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) @@ -23,3 +25,6 @@ assert repr(obj2) == "" assert repr(module.fooType.__call__) == "" assert obj2(foo=1, bar=2) == dict(foo=1, bar=2) + + print obj.foo + assert obj.foo == 42 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 1 04:23:19 2010 @@ -1,5 +1,6 @@ import os import sys +from weakref import ref from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.annlowlevel import llhelper @@ -103,6 +104,12 @@ def __init__(self, space): self.space = space + def set_dual(self, w_obj): + self.w_dual = w_obj + + def __del__(self): + Py_DecRef(self.space, self) + def getter(self, space, w_self): return generic_cpy_call( space, self.getset.c_get, w_self, @@ -113,6 +120,17 @@ space, self.getset.c_set, w_self, w_value, self.getset.c_closure) + +class W_PyCObjectDual(W_PyCObject): + def __init__(self, space): + self.space = space + + def set_pycobject(self, w_pycobject): + self.w_pycobject = ref(w_pycobject) + + def __del__(self): # ok, subclassing isnt so sensible here + pass + @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) From xoraxax at codespeak.net Thu Apr 1 04:41:16 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 04:41:16 +0200 (CEST) Subject: [pypy-svn] r73239 - pypy/branch/cpython-extension/pypy/translator Message-ID: <20100401024116.E3B1A282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 04:41:15 2010 New Revision: 73239 Modified: pypy/branch/cpython-extension/pypy/translator/driver.py Log: Do not complete the annotation after the processing of every secondary entrypoint. Modified: pypy/branch/cpython-extension/pypy/translator/driver.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/driver.py (original) +++ pypy/branch/cpython-extension/pypy/translator/driver.py Thu Apr 1 04:41:15 2010 @@ -320,7 +320,7 @@ for func, inputtypes in self.secondary_entrypoints: if inputtypes == Ellipsis: continue - rettype = annotator.build_types(func, inputtypes) + rettype = annotator.build_types(func, inputtypes, False) if self.entry_point: s = annotator.build_types(self.entry_point, self.inputtypes) From xoraxax at codespeak.net Thu Apr 1 15:35:06 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 15:35:06 +0200 (CEST) Subject: [pypy-svn] r73243 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401133506.9A7BE282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 15:35:05 2010 New Revision: 73243 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add more typeobjects to the GLOBALS. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 1 15:35:05 2010 @@ -213,18 +213,20 @@ FUNCTIONS_STATIC = {} FUNCTIONS_C = {} TYPES = {} -GLOBALS = { +GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur 'Py_None': ('PyObject*', 'space.w_None'), 'Py_True': ('PyObject*', 'space.w_True'), 'Py_False': ('PyObject*', 'space.w_False'), - 'PyType_Type#': ('PyTypeObject*', 'space.w_type'), - 'PyBaseObject_Type#': ('PyTypeObject*', 'space.w_object'), } for exc_name in ['TypeError', 'ValueError', 'KeyError', 'Exception', 'BaseException']: GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.w_' + exc_name) +for cpyname, pypyexpr in [("Type", "space.w_type"), ("BaseObject", "space.w_object"), + ("Dict", "space.w_dict"), ("Tuple", "space.w_tuple")]: + GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) + def get_structtype_for_ctype(ctype): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] From xoraxax at codespeak.net Thu Apr 1 15:54:07 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 15:54:07 +0200 (CEST) Subject: [pypy-svn] r73244 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401135407.31CF0282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 15:54:05 2010 New Revision: 73244 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: ANN: cpyext is now translatable and can at least load and use the famous foo extension module! In this changeset, I just reformat the code and add the PyList_Type. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 1 15:54:05 2010 @@ -223,8 +223,12 @@ 'BaseException']: GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.w_' + exc_name) -for cpyname, pypyexpr in [("Type", "space.w_type"), ("BaseObject", "space.w_object"), - ("Dict", "space.w_dict"), ("Tuple", "space.w_tuple")]: +for cpyname, pypyexpr in {"Type": "space.w_type", + "BaseObject": "space.w_object", + "Dict": "space.w_dict", + "Tuple": "space.w_tuple", + "List": "space.w_list", + }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) def get_structtype_for_ctype(ctype): From xoraxax at codespeak.net Thu Apr 1 16:00:14 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 16:00:14 +0200 (CEST) Subject: [pypy-svn] r73245 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401140014.634D8282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 16:00:12 2010 New Revision: 73245 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Update TODO list. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Thu Apr 1 16:00:12 2010 @@ -11,13 +11,13 @@ - replace @cpython_api(external=False) by another explicit name: all it does is a lltype function pointer, no C code involved. - - - Generate decl code for global objects. - - - Make code translatable. - Implement Members. + - Fix distutil's build_ext to work with cpyext. + + - Extend the import machinery to load .so/.pyd files. + - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. From trundle at codespeak.net Thu Apr 1 16:11:09 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 1 Apr 2010 16:11:09 +0200 (CEST) Subject: [pypy-svn] r73246 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100401141109.C9A74282BD8@codespeak.net> Author: trundle Date: Thu Apr 1 16:11:08 2010 New Revision: 73246 Added: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Log: Add structmembers; only int members are implemented for now. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Thu Apr 1 16:11:08 2010 @@ -42,6 +42,9 @@ #include "dictobject.h" #include "intobject.h" +// XXX This shouldn't be included here +#include "structmember.h" + #include #endif Added: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Thu Apr 1 16:11:08 2010 @@ -0,0 +1,38 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext import structmemberdefs +from pypy.module.cpyext.api import ADDR, cpython_api +from pypy.module.cpyext.intobject import PyInt_AsLong +from pypy.module.cpyext.pyerrors import PyErr_Occurred +from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.typeobjectdefs import PyMemberDef + + + at cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject) +def PyMember_GetOne(space, obj, w_member): + ptr = rffi.cast(ADDR, obj) + member_type = w_member.c_type + if member_type == structmemberdefs.T_INT: + result = rffi.cast(rffi.INTP, ptr + w_member.c_offset) + w_result = space.wrap(result[0]) + else: + raise OperationError(space.w_SystemError, + space.wrap("bad memberdescr type")) + return w_result + + + at cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, error=-1) +def PyMember_SetOne(space, obj, w_member, w_value): + ptr = rffi.cast(ADDR, obj) + member_type = w_member.c_type + if member_type == structmemberdefs.T_INT: + w_long_value = PyInt_AsLong(space, w_value) + if w_long_value == -1 and PyErr_Occurred(space): + return -1 + array = rffi.cast(rffi.INTP, ptr + w_member.c_offset) + array[0] = rffi.cast(rffi.INT, w_long_value) + else: + raise OperationError(space.w_SystemError, + space.wrap("bad memberdescr type")) + return 0 Added: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Thu Apr 1 16:11:08 2010 @@ -0,0 +1,3 @@ +T_INT = 1 + +READONLY = RO = 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Thu Apr 1 16:11:08 2010 @@ -1,4 +1,5 @@ #include "Python.h" +#include "structmember.h" typedef struct { PyObject_HEAD @@ -92,6 +93,15 @@ return kwds; } +static PyMemberDef foo_members[] = { + {"int_member", T_INT, offsetof(fooobject, foo), 0, + "A helpful docstring."}, + {"int_member_readonly", T_INT, offsetof(fooobject, foo), READONLY, + "A helpful docstring."}, + {"broken_member", 0xaffe, 0, 0, ""}, + {NULL} /* Sentinel */ +}; + static PyTypeObject footype = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", /*tp_name*/ @@ -122,7 +132,7 @@ 0, /*tp_iter*/ 0, /*tp_iternext*/ foo_methods, /*tp_methods*/ - 0, /*tp_members*/ + foo_members, /*tp_members*/ foo_getseters, /*tp_getset*/ }; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Thu Apr 1 16:11:08 2010 @@ -28,3 +28,9 @@ print obj.foo assert obj.foo == 42 + assert obj.int_member == obj.foo + obj.int_member = 23 + assert obj.int_member == 23 + raises(TypeError, "obj.int_member_readonly = 42") + raises(SystemError, "obj.broken_member") + raises(SystemError, "obj.broken_member = 42") Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 1 16:11:08 2010 @@ -15,12 +15,14 @@ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module +from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.state import State from pypy.module.cpyext.methodobject import PyDescr_NewWrapper from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef +from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ - PyGetSetDef + PyGetSetDef, PyMemberDef from pypy.module.cpyext.slotdefs import slotdefs from pypy.rlib.rstring import rsplit @@ -42,6 +44,19 @@ def PyDescr_NewGetSet(space, getset, pto): return space.wrap(W_GetSetPropertyEx(getset)) +class W_MemberDescr(GetSetProperty): + def __init__(self, member): + self.member = member + self.name = rffi.charp2str(member.c_name) + doc = set = None + if doc: + doc = rffi.charp2str(getset.c_doc) + get = W_PyCObject.member_getter + if not (member.c_flags & structmemberdefs.READONLY): + set = W_PyCObject.member_setter + GetSetProperty.__init__(self, get, set, None, doc, + cls=W_PyCObject, use_closure=True) + def convert_getset_defs(space, dict_w, getsets, pto): getsets = rffi.cast(rffi.CArrayPtr(PyGetSetDef), getsets) if getsets: @@ -56,6 +71,20 @@ w_descr = PyDescr_NewGetSet(space, getset, pto) dict_w[name] = w_descr +def convert_member_defs(space, dict_w, members, pto): + members = rffi.cast(rffi.CArrayPtr(PyMemberDef), members) + if members: + i = 0 + while True: + member = members[i] + name = member.c_name + if not name: + break + name = rffi.charp2str(name) + w_descr = space.wrap(W_MemberDescr(member)) + dict_w[name] = w_descr + i += 1 + def add_operators(space, dict_w, pto): # XXX support PyObject_HashNotImplemented state = space.fromcache(State) @@ -90,7 +119,7 @@ add_operators(space, dict_w, pto) convert_method_defs(space, dict_w, pto.c_tp_methods, pto) convert_getset_defs(space, dict_w, pto.c_tp_getset, pto) - # XXX missing: convert_member_defs + convert_member_defs(space, dict_w, pto.c_tp_members, pto) full_name = rffi.charp2str(pto.c_tp_name) module_name, extension_name = rsplit(full_name, ".", 1) @@ -120,6 +149,11 @@ space, self.getset.c_set, w_self, w_value, self.getset.c_closure) + def member_getter(self, space, w_self): + return PyMember_GetOne(space, w_self, self.member) + + def member_setter(self, space, w_self, w_value): + PyMember_SetOne(space, w_self, self.member, w_value) class W_PyCObjectDual(W_PyCObject): def __init__(self, space): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Thu Apr 1 16:11:08 2010 @@ -15,7 +15,7 @@ # XXX -PyBufferProcs = PyMemberDef = rffi.VOIDP.TO +PyBufferProcs = rffi.VOIDP.TO freefunc = P(FT([rffi.VOIDP_real], Void)) destructor = P(FT([PyO], Void)) @@ -144,6 +144,14 @@ )) """ +PyMemberDef = cpython_struct("PyMemberDef", ( + ("name", rffi.CCHARP), + ("type", rffi.INT_real), + ("offset", Py_ssize_t), + ("flags", rffi.INT_real), + ("doc", rffi.CCHARP), +)) + PyTypeObjectFields = [] PyTypeObjectFields.extend(PyVarObjectFields) PyTypeObjectFields.extend([ From xoraxax at codespeak.net Thu Apr 1 16:25:54 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 1 Apr 2010 16:25:54 +0200 (CEST) Subject: [pypy-svn] r73247 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401142554.17E8C282BD8@codespeak.net> Author: xoraxax Date: Thu Apr 1 16:25:52 2010 New Revision: 73247 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: New TODO items. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Thu Apr 1 16:25:52 2010 @@ -1,4 +1,9 @@ - - Complete the PyTypeObject initialization code. + - Complete the PyTypeObject initialization code. (see XXX in the code) + - Fill the PyTypeObject slots with method callers. + - Copy the slots from the base. + - Both tasks are necessary to be able to call slots from C code correctly. + + - Implement subclassing of types defined in C. - Use a WeakKeyDictionary to count how often a PyObject is allocated for a given wrapped object and use this to assess whether optimizations are From trundle at codespeak.net Thu Apr 1 16:28:05 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 1 Apr 2010 16:28:05 +0200 (CEST) Subject: [pypy-svn] r73248 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100401142805.8588D282BD8@codespeak.net> Author: trundle Date: Thu Apr 1 16:28:04 2010 New Revision: 73248 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Remove wrong error checking and add a test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Thu Apr 1 16:28:04 2010 @@ -28,8 +28,6 @@ member_type = w_member.c_type if member_type == structmemberdefs.T_INT: w_long_value = PyInt_AsLong(space, w_value) - if w_long_value == -1 and PyErr_Occurred(space): - return -1 array = rffi.cast(rffi.INTP, ptr + w_member.c_offset) array[0] = rffi.cast(rffi.INT, w_long_value) else: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Thu Apr 1 16:28:04 2010 @@ -31,6 +31,8 @@ assert obj.int_member == obj.foo obj.int_member = 23 assert obj.int_member == 23 + obj.int_member = 42 + raises(TypeError, "obj.int_member = 'not a number'") raises(TypeError, "obj.int_member_readonly = 42") raises(SystemError, "obj.broken_member") raises(SystemError, "obj.broken_member = 42") From trundle at codespeak.net Thu Apr 1 16:32:47 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 1 Apr 2010 16:32:47 +0200 (CEST) Subject: [pypy-svn] r73249 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401143247.41ABA282BD8@codespeak.net> Author: trundle Date: Thu Apr 1 16:32:45 2010 New Revision: 73249 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Add cast to make it RPython. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 1 16:32:45 2010 @@ -48,11 +48,12 @@ def __init__(self, member): self.member = member self.name = rffi.charp2str(member.c_name) + flags = rffi.cast(lltype.Signed, member.c_flags) doc = set = None if doc: doc = rffi.charp2str(getset.c_doc) get = W_PyCObject.member_getter - if not (member.c_flags & structmemberdefs.READONLY): + if not (flags & structmemberdefs.READONLY): set = W_PyCObject.member_setter GetSetProperty.__init__(self, get, set, None, doc, cls=W_PyCObject, use_closure=True) From afa at codespeak.net Thu Apr 1 16:34:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Apr 2010 16:34:02 +0200 (CEST) Subject: [pypy-svn] r73250 - in pypy/trunk/pypy: module/pyexpat module/pyexpat/test rpython/lltypesystem rpython/lltypesystem/test translator/c Message-ID: <20100401143402.54A23282BD8@codespeak.net> Author: afa Date: Thu Apr 1 16:34:00 2010 New Revision: 73250 Modified: pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py pypy/trunk/pypy/module/pyexpat/test/test_build.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py pypy/trunk/pypy/translator/c/database.py Log: with headers like: typedef struct _opaque HANDLE; Implement a way to use HANDLE in the generated C code: HANDLE = rffi.COpaquePtr(typedef='HANDLE') Use it in pyexpat. Modified: pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/trunk/pypy/module/pyexpat/interp_pyexpat.py Thu Apr 1 16:34:00 2010 @@ -33,7 +33,7 @@ ]) XML_Content_Ptr = lltype.Ptr(lltype.ForwardReference()) -XML_Parser = rffi.VOIDP # an opaque pointer +XML_Parser = rffi.COpaquePtr(typedef='XML_Parser') class CConfigure: _compilation_info_ = eci @@ -475,7 +475,7 @@ global_storage.get_nonmoving_id( CallbackData(space, parser), id=rffi.cast(lltype.Signed, parser.itself)) - XML_SetUserData(parser.itself, parser.itself) + XML_SetUserData(parser.itself, rffi.cast(rffi.VOIDP, parser.itself)) # copy handlers from self for i in range(NB_HANDLERS): @@ -621,7 +621,7 @@ global_storage.get_nonmoving_id( CallbackData(space, parser), id=rffi.cast(lltype.Signed, parser.itself)) - XML_SetUserData(parser.itself, parser.itself) + XML_SetUserData(parser.itself, rffi.cast(rffi.VOIDP, parser.itself)) return space.wrap(parser) ParserCreate.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] Modified: pypy/trunk/pypy/module/pyexpat/test/test_build.py ============================================================================== --- pypy/trunk/pypy/module/pyexpat/test/test_build.py (original) +++ pypy/trunk/pypy/module/pyexpat/test/test_build.py Thu Apr 1 16:34:00 2010 @@ -18,6 +18,8 @@ def test_build(): def entry_point(argv): + parser = interp_pyexpat.XML_ParserCreate("test") + interp_pyexpat.XML_ParserFree(parser) res = interp_pyexpat.XML_ErrorString(3) os.write(1, rffi.charp2str(res)) return 0 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Thu Apr 1 16:34:00 2010 @@ -448,7 +448,7 @@ return lltype.Ptr(lltype.FuncType(args, res)) CCallback._annspecialcase_ = 'specialize:memo' -def COpaque(name, hints=None, compilation_info=None): +def COpaque(name=None, ptr_typedef=None, hints=None, compilation_info=None): if compilation_info is None: compilation_info = ExternalCompilationInfo() if hints is None: @@ -456,7 +456,10 @@ else: hints = hints.copy() hints['external'] = 'C' - hints['c_name'] = name + if name is not None: + hints['c_name'] = name + if ptr_typedef is not None: + hints['c_pointer_typedef'] = ptr_typedef def lazy_getsize(cache={}): from pypy.rpython.tool import rffi_platform try: @@ -470,7 +473,8 @@ return lltype.OpaqueType(name, hints) def COpaquePtr(*args, **kwds): - return lltype.Ptr(COpaque(*args, **kwds)) + typedef = kwds.pop('typedef', None) + return lltype.Ptr(COpaque(ptr_typedef=typedef, *args, **kwds)) def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant, sandboxsafe=False, _nowrapper=False, Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_rffi.py Thu Apr 1 16:34:00 2010 @@ -280,6 +280,28 @@ f1 = self.compile(f, []) assert f1() == 'a' + def test_opaque_typedef(self): + code = """ + #include + struct stuff; + typedef struct stuff *stuff_ptr; + static int get(stuff_ptr ptr) { return (ptr != NULL); } + """ + + eci = ExternalCompilationInfo( + post_include_bits = [code] + ) + + STUFFP = COpaquePtr(typedef='stuff_ptr', compilation_info=eci) + ll_get = llexternal('get', [STUFFP], lltype.Signed, + compilation_info=eci) + + def f(): + return ll_get(lltype.nullptr(STUFFP.TO)) + + f1 = self.compile(f, []) + assert f1() == 0 + def return_char(self, signed): ctype_pref = ["un", ""][signed] rffi_type = [UCHAR, SIGNEDCHAR][signed] Modified: pypy/trunk/pypy/translator/c/database.py ============================================================================== --- pypy/trunk/pypy/translator/c/database.py (original) +++ pypy/trunk/pypy/translator/c/database.py Thu Apr 1 16:34:00 2010 @@ -101,6 +101,9 @@ if isinstance(T, Primitive) or T == GCREF: return PrimitiveType[T] elif isinstance(T, Ptr): + if (isinstance(T.TO, OpaqueType) and + T.TO.hints.get('c_pointer_typedef') is not None): + return '%s @' % T.TO.hints['c_pointer_typedef'] try: node = self.gettypedefnode(T.TO) except NoCorrespondingNode: From trundle at codespeak.net Thu Apr 1 16:40:49 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 1 Apr 2010 16:40:49 +0200 (CEST) Subject: [pypy-svn] r73251 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100401144049.519DC282BD8@codespeak.net> Author: trundle Date: Thu Apr 1 16:40:47 2010 New Revision: 73251 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Log: Add forgotten structmember.h, sorry. Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Thu Apr 1 16:40:47 2010 @@ -0,0 +1,29 @@ +#ifndef Py_STRUCTMEMBER_H +#define Py_STRUCTMEMBER_H +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct PyMemberDef { + /* Current version, use this */ + char *name; + int type; + Py_ssize_t offset; + int flags; + char *doc; +} PyMemberDef; + + +/* Types */ +#define T_INT 1 + +/* Flags */ +#define READONLY 1 +#define RO READONLY /* Shorthand */ + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_STRUCTMEMBER_H */ From fijal at codespeak.net Thu Apr 1 18:26:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Apr 2010 18:26:40 +0200 (CEST) Subject: [pypy-svn] r73252 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100401162640.D3FCF282BD8@codespeak.net> Author: fijal Date: Thu Apr 1 18:26:36 2010 New Revision: 73252 Added: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Add PyEval_CallObject[WithKeywords] Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 1 18:26:36 2010 @@ -44,5 +44,6 @@ import pypy.module.cpyext.intobject import pypy.module.cpyext.listobject import pypy.module.cpyext.sequence +import pypy.module.cpyext.eval # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/eval.py Thu Apr 1 18:26:36 2010 @@ -0,0 +1,6 @@ + +from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL + + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): + return space.call(w_obj, w_arg, w_kwds) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Thu Apr 1 18:26:36 2010 @@ -41,6 +41,7 @@ #include "tupleobject.h" #include "dictobject.h" #include "intobject.h" +#include "eval.h" // XXX This shouldn't be included here #include "structmember.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h Thu Apr 1 18:26:36 2010 @@ -0,0 +1,18 @@ + +/* Int object interface */ + +#ifndef Py_EVAL_H +#define Py_EVAL_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "Python.h" + +#define PyEval_CallObject(func,arg) \ + PyEval_CallObjectWithKeywords(func, arg, (PyObject *)NULL) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_EVAL_H */ Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Thu Apr 1 18:26:36 2010 @@ -0,0 +1,30 @@ + +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestEval(BaseApiTest): + def test_eval(self, space, api): + w_l, w_f = space.fixedview(space.appexec([], """(): + l = [] + def f(arg1, arg2): + l.append(arg1) + l.append(arg2) + return len(l) + return l, f + """)) + + w_t = space.newtuple([space.wrap(1), space.wrap(2)]) + w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, None) + assert space.int_w(w_res) == 2 + assert space.int_w(space.len(w_l)) == 2 + w_f = space.appexec([], """(): + def f(*args, **kwds): + assert isinstance(kwds, dict) + assert 'xyz' in kwds + return len(kwds) + len(args) * 10 + return f + """) + w_t = space.newtuple([space.w_None, space.w_None]) + w_d = space.newdict() + space.setitem(w_d, space.wrap("xyz"), space.wrap(3)) + w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d) + assert space.int_w(w_res) == 21 From benjamin at codespeak.net Thu Apr 1 19:43:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 1 Apr 2010 19:43:29 +0200 (CEST) Subject: [pypy-svn] r73253 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401174329.E6AE9282BD8@codespeak.net> Author: benjamin Date: Thu Apr 1 19:43:27 2010 New Revision: 73253 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/boolobject.py pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py Log: generalize general_check and general_check_exact into a build_type_checkers function Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 1 19:43:27 2010 @@ -114,7 +114,8 @@ wrapper.relax_sig_check = True return wrapper -def cpython_api(argtypes, restype, borrowed=False, error=_NOT_SPECIFIED, external=True): +def cpython_api(argtypes, restype, borrowed=False, error=_NOT_SPECIFIED, + external=True, name=None): if error is _NOT_SPECIFIED: if restype is PyObject: error = lltype.nullptr(PyObject.TO) @@ -126,6 +127,10 @@ def decorate(func): api_function = ApiFunction(argtypes, restype, func, borrowed, error) func.api_func = api_function + if name is None: + func_name = func.func_name + else: + func_name = name if error is _NOT_SPECIFIED: raise ValueError("function %s has no return value for exceptions" @@ -134,7 +139,7 @@ names = api_function.argnames types_names_enum_ui = unrolling_iterable(enumerate( zip(api_function.argtypes, - [name.startswith("w_") for name in names]))) + [tp_name.startswith("w_") for tp_name in names]))) @specialize.ll() def unwrapper(space, *args): @@ -188,10 +193,10 @@ unwrapper_catch = make_unwrapper(True) unwrapper_raise = make_unwrapper(False) if external: - FUNCTIONS[func.func_name] = api_function + FUNCTIONS[func_name] = api_function else: - FUNCTIONS_STATIC[func.func_name] = api_function - INTERPLEVEL_API[func.func_name] = unwrapper_catch # used in tests + FUNCTIONS_STATIC[func_name] = api_function + INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests return unwrapper_raise # used in 'normal' RPython code. return decorate @@ -255,14 +260,23 @@ if name in TYPES: TYPES[name].become(TYPE) - -def general_check(space, w_obj, w_type): - w_obj_type = space.type(w_obj) - return int(space.is_w(w_obj_type, w_type) or space.is_true(space.issubtype(w_obj_type, w_type))) - -def general_check_exact(space, w_obj, w_type): - w_obj_type = space.type(w_obj) - return int(space.is_w(w_obj_type, w_type)) +def build_type_checkers(type_name, on_space=None): + if on_space is None: + on_space = "w_" + type_name.lower() + check_name = "Py" + type_name + "_Check" + @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, name=check_name) + def check(space, w_obj): + w_obj_type = space.type(w_obj) + w_type = getattr(space, on_space) + return int(space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))) + @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, + name=check_name + "Exact") + def check_exact(space, w_obj): + w_obj_type = space.type(w_obj) + w_type = getattr(space, on_space) + return int(space.is_w(w_obj_type, w_type)) + return check, check_exact # Make the wrapper for the cases (1) and (2) def make_wrapper(space, callable): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/boolobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/boolobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/boolobject.py Thu Apr 1 19:43:27 2010 @@ -1,11 +1,9 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL -from pypy.module.cpyext.api import general_check +from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, + build_type_checkers) - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyBool_Check(space, w_obj): - w_type = space.w_bool - return general_check(space, w_obj, w_type) +# Inheriting from bool isn't actually possible. +PyBool_Check = build_type_checkers("Bool")[1] @cpython_api([rffi.LONG], PyObject) def PyBool_FromLong(space, value): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Thu Apr 1 19:43:27 2010 @@ -1,6 +1,5 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL -from pypy.module.cpyext.api import general_check, general_check_exact +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -9,15 +8,7 @@ def PyDict_New(space): return space.newdict() - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyDict_Check(space, w_obj): - w_type = space.w_dict - return general_check(space, w_obj, w_type) - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyDict_CheckExact(space, w_obj): - w_type = space.w_dict - return general_check_exact(space, w_obj, w_type) +PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") @cpython_api([PyObject, PyObject], PyObject) def PyDict_GetItem(space, w_dict, w_key): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Thu Apr 1 19:43:27 2010 @@ -1,15 +1,10 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL -from pypy.module.cpyext.api import general_check +from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, + build_type_checkers) - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyInt_Check(space, w_obj): - """Return true if o is of type PyInt_Type or a subtype of - PyInt_Type. - - Allowed subtypes to be accepted.""" - return general_check(space, w_obj, space.w_int) + +PyInt_Check, PyInt_CheckExact = build_type_checkers("Int") @cpython_api([lltype.Signed], PyObject) def PyInt_FromLong(space, ival): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Thu Apr 1 19:43:27 2010 @@ -1,28 +1,14 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t -from pypy.module.cpyext.api import general_check, general_check_exact +from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, + build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.pyobject import Py_DecRef, PyObject from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyList_Check(space, w_obj): - """Return true if p is a list object or an instance of a subtype of the list - type. - - Allowed subtypes to be accepted.""" - w_type = space.w_list - return general_check(space, w_obj, w_type) - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyList_CheckExact(space, w_obj): - """Return true if p is a list object, but not an instance of a subtype of - the list type. - """ - w_type = space.w_list - return general_check_exact(space, w_obj, w_type) + +PyList_Check, PyList_CheckExact = build_type_checkers("List") @cpython_api([Py_ssize_t], PyObject) def PyList_New(space, len): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Thu Apr 1 19:43:27 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, general_check, CANNOT_FAIL + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import PyCFunction_NewEx, PyDescr_NewMethod @@ -77,7 +77,9 @@ @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyModule_Check(space, w_obj): w_type = space.gettypeobject(Module.typedef) - return general_check(space, w_obj, w_type) + w_obj_type = space.type(w_obj) + return int(space.is_w(w_type, w_obj_type) or + space.issubtype(w_obj_type, w_type)) @cpython_api([PyObject], PyObject, borrowed=True) def PyModule_GetDict(space, w_mod): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Thu Apr 1 19:43:27 2010 @@ -1,17 +1,11 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyVarObjectFields, \ - PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, general_check +from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, + PyStringObject, Py_ssize_t, cpython_struct, + CANNOT_FAIL, build_type_checkers) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyString_Check(space, w_obj): - """Return true if the object o is a string object or an instance of a subtype of - the string type. - - Allowed subtypes to be accepted.""" - w_type = space.w_str - return general_check(space, w_obj, w_type) +PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") @cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py Thu Apr 1 19:43:27 2010 @@ -1,19 +1,12 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, Py_ssize_t, \ - general_check, CANNOT_FAIL, general_check_exact +from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, + build_type_checkers) from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, register_container from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTuple_Check(space, w_obj): - w_type = space.w_tuple - return general_check(space, w_obj, w_type) - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTuple_CheckExact(space, w_obj): - w_type = space.w_tuple - return general_check_exact(space, w_obj, w_type) + +PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): From afa at codespeak.net Thu Apr 1 22:53:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Apr 2010 22:53:41 +0200 (CEST) Subject: [pypy-svn] r73256 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401205341.295E3282BE2@codespeak.net> Author: afa Date: Thu Apr 1 22:53:39 2010 New Revision: 73256 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Log: One translation fix Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Thu Apr 1 22:53:39 2010 @@ -79,7 +79,7 @@ w_type = space.gettypeobject(Module.typedef) w_obj_type = space.type(w_obj) return int(space.is_w(w_type, w_obj_type) or - space.issubtype(w_obj_type, w_type)) + space.is_true(space.issubtype(w_obj_type, w_type))) @cpython_api([PyObject], PyObject, borrowed=True) def PyModule_GetDict(space, w_mod): From fijal at codespeak.net Thu Apr 1 23:22:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Apr 2010 23:22:40 +0200 (CEST) Subject: [pypy-svn] r73258 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100401212240.BCE8E282BD8@codespeak.net> Author: fijal Date: Thu Apr 1 23:22:39 2010 New Revision: 73258 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (contents, props changed) Log: A failing test Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Thu Apr 1 23:22:39 2010 @@ -0,0 +1,17 @@ + +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestGetargs(AppTestCpythonExtensionBase): + def test_pyarg_parse(self): + mod = self.import_extension('foo', [ + ('oneargint', 'METH_VARARGS', + ''' + int l; + #if (!PyArg_Parse(args, "i", &l)) { + # return NULL; + #} + return PyInt_FromLong(l); + ''' + )]) + assert mod.oneargint(1) == 1 From fijal at codespeak.net Thu Apr 1 23:26:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Apr 2010 23:26:53 +0200 (CEST) Subject: [pypy-svn] r73259 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100401212653.D77BC282BD8@codespeak.net> Author: fijal Date: Thu Apr 1 23:26:52 2010 New Revision: 73259 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: use different comment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Thu Apr 1 23:26:52 2010 @@ -8,9 +8,9 @@ ('oneargint', 'METH_VARARGS', ''' int l; - #if (!PyArg_Parse(args, "i", &l)) { - # return NULL; - #} + //if (!PyArg_Parse(args, "i", &l)) { + // return NULL; + //} return PyInt_FromLong(l); ''' )]) From afa at codespeak.net Fri Apr 2 00:35:47 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 00:35:47 +0200 (CEST) Subject: [pypy-svn] r73262 - pypy/branch/cpython-extension/pypy/interpreter Message-ID: <20100401223547.CA233282BD8@codespeak.net> Author: afa Date: Fri Apr 2 00:35:46 2010 New Revision: 73262 Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py Log: Finally managed it to translate when several families of GetSetProperty can be created at runtime Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/typedef.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/typedef.py Fri Apr 2 00:35:46 2010 @@ -301,22 +301,23 @@ # ____________________________________________________________ - at specialize.arg(1) -def make_descr_typecheck_wrapper(func, extraargs=(), cls=None, use_closure=False): + at specialize.arg(0, 2) +def make_descr_typecheck_wrapper(tag, func, extraargs=(), cls=None, + use_closure=False): if func is None: return None - return _make_descr_typecheck_wrapper(func, extraargs, cls, use_closure) + return _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure) @specialize.memo() -def _make_descr_typecheck_wrapper(func, extraargs, cls, use_closure): +def _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure): # - if cls is None, the wrapped object is passed to the function # - if cls is a class, an unwrapped instance is passed # - if cls is a string, XXX unused? if cls is None and use_closure: return func - if hasattr(func, 'im_func'): - assert func.im_class is cls - func = func.im_func + #if hasattr(func, 'im_func'): + # assert func.im_class is cls + # func = func.im_func miniglobals = { func.__name__: func, @@ -366,8 +367,9 @@ raise OperationError(space.w_AttributeError, space.wrap("generic property has no __objclass__")) -def make_objclass_getter(func, cls): - if func and hasattr(func, 'im_func'): + at specialize.arg(0) +def make_objclass_getter(tag, func, cls): + if func and not cls and hasattr(func, 'im_func'): assert not cls or cls is func.im_class cls = func.im_class return _make_objclass_getter(cls) @@ -393,14 +395,15 @@ return res class GetSetProperty(Wrappable): + @specialize.arg(7) def __init__(self, fget, fset=None, fdel=None, doc=None, - cls=None, use_closure=False): - objclass_getter, cls = make_objclass_getter(fget, cls) - fget = make_descr_typecheck_wrapper(fget, cls=cls, + cls=None, use_closure=False, tag=None): + objclass_getter, cls = make_objclass_getter(tag, fget, cls) + fget = make_descr_typecheck_wrapper(tag, fget, cls=cls, use_closure=use_closure) - fset = make_descr_typecheck_wrapper(fset, ('w_value',), cls=cls, + fset = make_descr_typecheck_wrapper(tag, fset, ('w_value',), cls=cls, use_closure=use_closure) - fdel = make_descr_typecheck_wrapper(fdel, cls=cls, + fdel = make_descr_typecheck_wrapper(tag, fdel, cls=cls, use_closure=use_closure) self.fget = fget self.fset = fset From afa at codespeak.net Fri Apr 2 00:50:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 00:50:13 +0200 (CEST) Subject: [pypy-svn] r73263 - pypy/branch/cpython-extension/pypy/annotation Message-ID: <20100401225013.EA45F282BD8@codespeak.net> Author: afa Date: Fri Apr 2 00:50:12 2010 New Revision: 73263 Modified: pypy/branch/cpython-extension/pypy/annotation/description.py Log: This is also needed for specialise.memo() when there is a function in the arguments Modified: pypy/branch/cpython-extension/pypy/annotation/description.py ============================================================================== --- pypy/branch/cpython-extension/pypy/annotation/description.py (original) +++ pypy/branch/cpython-extension/pypy/annotation/description.py Fri Apr 2 00:50:12 2010 @@ -169,6 +169,8 @@ class NoStandardGraph(Exception): """The function doesn't have a single standard non-specialized graph.""" +NODEFAULT = object() + class FunctionDesc(Desc): knowntype = types.FunctionType overridden = False @@ -191,6 +193,7 @@ # or => s_result (overridden/memo cases) self.specializer = specializer self._cache = {} # convenience for the specializer + self.memofield = {} def buildgraph(self, alt_name=None, builder=None): translator = self.bookkeeper.annotator.translator @@ -355,7 +358,27 @@ return s_sigs -NODEFAULT = object() + def create_new_attribute(self, name, value): + assert name.startswith('$memofield_') + self.memofield[name] = value + + def read_attribute(self, name, default=NODEFAULT): + assert name.startswith('$memofield_') + try: + return self.memofield[name] + except: + if default is not NODEFAULT: + return default + else: + raise AttributeError + + def s_read_attribute(self, attr): + from pypy.annotation.model import s_ImpossibleValue + return s_ImpossibleValue + + def mergeattrfamilies(self, others, attrname): + # no attr to merge + return False class ClassDesc(Desc): knowntype = type From afa at codespeak.net Fri Apr 2 01:02:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 01:02:51 +0200 (CEST) Subject: [pypy-svn] r73264 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401230251.2447D282B9C@codespeak.net> Author: afa Date: Fri Apr 2 01:02:49 2010 New Revision: 73264 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: This is the piece which allows distinct specialisations of functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 2 01:02:49 2010 @@ -39,7 +39,8 @@ if getset.c_set: set = W_PyCObject.setter GetSetProperty.__init__(self, get, set, None, doc, - cls=W_PyCObject, use_closure=True) + cls=W_PyCObject, use_closure=True, + tag="cpyext_1") def PyDescr_NewGetSet(space, getset, pto): return space.wrap(W_GetSetPropertyEx(getset)) @@ -56,7 +57,8 @@ if not (flags & structmemberdefs.READONLY): set = W_PyCObject.member_setter GetSetProperty.__init__(self, get, set, None, doc, - cls=W_PyCObject, use_closure=True) + cls=W_PyCObject, use_closure=True, + tag="cpyext_2") def convert_getset_defs(space, dict_w, getsets, pto): getsets = rffi.cast(rffi.CArrayPtr(PyGetSetDef), getsets) From afa at codespeak.net Fri Apr 2 01:24:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 01:24:14 +0200 (CEST) Subject: [pypy-svn] r73265 - pypy/branch/cpython-extension/pypy/annotation Message-ID: <20100401232414.09629282B9C@codespeak.net> Author: afa Date: Fri Apr 2 01:24:13 2010 New Revision: 73265 Modified: pypy/branch/cpython-extension/pypy/annotation/description.py Log: These changes were not necessary and most probably wrong, revert them. Modified: pypy/branch/cpython-extension/pypy/annotation/description.py ============================================================================== --- pypy/branch/cpython-extension/pypy/annotation/description.py (original) +++ pypy/branch/cpython-extension/pypy/annotation/description.py Fri Apr 2 01:24:13 2010 @@ -169,8 +169,6 @@ class NoStandardGraph(Exception): """The function doesn't have a single standard non-specialized graph.""" -NODEFAULT = object() - class FunctionDesc(Desc): knowntype = types.FunctionType overridden = False @@ -193,7 +191,6 @@ # or => s_result (overridden/memo cases) self.specializer = specializer self._cache = {} # convenience for the specializer - self.memofield = {} def buildgraph(self, alt_name=None, builder=None): translator = self.bookkeeper.annotator.translator @@ -358,27 +355,7 @@ return s_sigs - def create_new_attribute(self, name, value): - assert name.startswith('$memofield_') - self.memofield[name] = value - - def read_attribute(self, name, default=NODEFAULT): - assert name.startswith('$memofield_') - try: - return self.memofield[name] - except: - if default is not NODEFAULT: - return default - else: - raise AttributeError - - def s_read_attribute(self, attr): - from pypy.annotation.model import s_ImpossibleValue - return s_ImpossibleValue - - def mergeattrfamilies(self, others, attrname): - # no attr to merge - return False +NODEFAULT = object() class ClassDesc(Desc): knowntype = type From xoraxax at codespeak.net Fri Apr 2 01:39:11 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 01:39:11 +0200 (CEST) Subject: [pypy-svn] r73266 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401233911.7A8B7282B9C@codespeak.net> Author: xoraxax Date: Fri Apr 2 01:39:09 2010 New Revision: 73266 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Log: Fix translation by adding two casts to the structmember code. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Fri Apr 2 01:39:09 2010 @@ -12,7 +12,7 @@ @cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject) def PyMember_GetOne(space, obj, w_member): ptr = rffi.cast(ADDR, obj) - member_type = w_member.c_type + member_type = rffi.cast(lltype.Signed, w_member.c_type) if member_type == structmemberdefs.T_INT: result = rffi.cast(rffi.INTP, ptr + w_member.c_offset) w_result = space.wrap(result[0]) @@ -25,7 +25,7 @@ @cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, error=-1) def PyMember_SetOne(space, obj, w_member, w_value): ptr = rffi.cast(ADDR, obj) - member_type = w_member.c_type + member_type = rffi.cast(lltype.Signed, w_member.c_type) if member_type == structmemberdefs.T_INT: w_long_value = PyInt_AsLong(space, w_value) array = rffi.cast(rffi.INTP, ptr + w_member.c_offset) From fijal at codespeak.net Fri Apr 2 01:41:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 01:41:44 +0200 (CEST) Subject: [pypy-svn] r73267 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100401234144.DA606282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 01:41:43 2010 New Revision: 73267 Added: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Add support for PyArg_Parse. Right now it happily segfaults Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Fri Apr 2 01:41:43 2010 @@ -45,5 +45,7 @@ import pypy.module.cpyext.listobject import pypy.module.cpyext.sequence import pypy.module.cpyext.eval +import pypy.module.cpyext.getargs + # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 01:41:43 2010 @@ -38,12 +38,14 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( include_dirs=include_dirs, - includes=['Python.h'] + includes=['Python.h', 'stdarg.h'] ) class CConfig_constants: _compilation_info_ = CConfig._compilation_info_ +va_list = rffi.COpaque('va_list') + constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS @@ -202,7 +204,10 @@ def cpython_api_c(): def decorate(func): + def uncallable(*args, **kwds): + raise Exception("Uncallable") FUNCTIONS_C[func.func_name] = None + return uncallable return decorate def cpython_struct(name, fields, forward=None): @@ -255,6 +260,9 @@ (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +VA_TP_LIST = {'int': rffi.INT, + 'PyObject*': PyObject} + def configure_types(): for name, TYPE in rffi_platform.configure(CConfig).iteritems(): if name in TYPES: @@ -434,6 +442,13 @@ ll2ctypes.lltype2ctypes(func.get_llhelper(space)), ctypes.c_void_p) + for name, TP in VA_TP_LIST.iteritems(): + name_no_star = name.strip('*') + ARGS = [lltype.Ptr(va_list)] + func = rffi.llexternal('pypy_va_get_%s' % name_no_star, ARGS, + TP, compilation_info=eci) + globals()['va_get_%s' % name_no_star] = func + return modulename.new(ext='') def generate_macros(export_symbols, rename=True, do_deref=True): @@ -482,6 +497,13 @@ pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, (typ, expr) in GLOBALS.iteritems(): pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) + for name in VA_TP_LIST: + name_no_star = name.strip('*') + header = ('%s pypy_va_get_%s(va_list* vp)' % + (name, name_no_star)) + pypy_decls.append(header + ';') + functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name) + pypy_decls.append("#endif\n") pypy_decl_h = udir.join('pypy_decl.h') @@ -510,7 +532,8 @@ include_dirs=include_dirs, separate_module_files=[include_dir / "varargwrapper.c", include_dir / "pyerrors.c", - include_dir / "modsupport.c"], + include_dir / "modsupport.c", + include_dir / "getargs.c"], export_symbols=export_symbols_eci, **kwds ) Added: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 01:41:43 2010 @@ -0,0 +1,24 @@ + +from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL, va_list,\ + cpython_api_c +from pypy.module.cpyext import api +from pypy.rpython.lltypesystem import lltype, rffi + + at cpython_api_c() +def PyArg_Parse(): + pass + + at cpython_api([PyObject, rffi.CCHARP, lltype.Ptr(va_list), rffi.INT_real], + rffi.INT_real, error=-1) +def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): + i = 0 + import pdb + pdb.set_trace() + while True: + c = fmt[i] + if c == "\x00": + return 0 + if c == "i": + pyobj = api.va_get_PyObject(va_list_p); + i += 1 + return 0 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c Fri Apr 2 01:41:43 2010 @@ -0,0 +1,141 @@ + +/* New getargs implementation */ + +#include "Python.h" + +#include + + +#ifdef __cplusplus +extern "C" { +#endif + +int PyArg_Parse(PyObject *, const char *, ...); +int PyArg_ParseTuple(PyObject *, const char *, ...); +int PyArg_VaParse(PyObject *, const char *, va_list); + +int PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, + const char *, char **, ...); +int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, + const char *, char **, va_list); + +#ifdef HAVE_DECLSPEC_DLL +/* Export functions */ +/*PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, char *, ...); +PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, char *, ...); +PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, + const char *, char **, ...); +PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); +PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, char *, va_list); +PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, +const char *, char **, va_list);*/ +#endif + +#define FLAG_COMPAT 1 +#define FLAG_SIZE_T 2 + + +/* Forward */ +int pypy_vgetargs1(struct _object *, char *, va_list *, int); + //static void seterror(int, const char *, int *, const char *, const char *); + //static char *convertitem(PyObject *, const char **, va_list *, int, int *, + // char *, size_t, PyObject **); +//static char *converttuple(PyObject *, const char **, va_list *, int, +// int *, char *, size_t, int, PyObject **); +//static char *convertsimple(PyObject *, const char **, va_list *, int, char *, +// size_t, PyObject **); +//static Py_ssize_t convertbuffer(PyObject *, void **p, char **); +//static int getbuffer(PyObject *, Py_buffer *, char**); + +//static int vgetargskeywords(PyObject *, PyObject *, +// const char *, char **, va_list *, int); +//static char *skipitem(const char **, va_list *, int); + +int +PyArg_Parse(PyObject *args, const char *format, ...) +{ + int retval; + va_list va; + + va_start(va, format); + retval = pypy_vgetargs1(args, format, &va, FLAG_COMPAT); + va_end(va); + return retval; +} + +int +_PyArg_Parse_SizeT(PyObject *args, char *format, ...) +{ + int retval; + va_list va; + + va_start(va, format); + retval = pypy_vgetargs1(args, format, &va, FLAG_COMPAT|FLAG_SIZE_T); + va_end(va); + return retval; +} + + +int +PyArg_ParseTuple(PyObject *args, const char *format, ...) +{ + int retval; + va_list va; + + va_start(va, format); + retval = pypy_vgetargs1(args, format, &va, 0); + va_end(va); + return retval; +} + +int +_PyArg_ParseTuple_SizeT(PyObject *args, char *format, ...) +{ + int retval; + va_list va; + + va_start(va, format); + retval = pypy_vgetargs1(args, format, &va, FLAG_SIZE_T); + va_end(va); + return retval; +} + + +int +PyArg_VaParse(PyObject *args, const char *format, va_list va) +{ + va_list lva; + +#ifdef VA_LIST_IS_ARRAY + memcpy(lva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(lva, va); +#else + lva = va; +#endif +#endif + + return pypy_vgetargs1(args, format, &lva, 0); +} + +int +_PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) +{ + va_list lva; + +#ifdef VA_LIST_IS_ARRAY + memcpy(lva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(lva, va); +#else + lva = va; +#endif +#endif + + return pypy_vgetargs1(args, format, &lva, FLAG_SIZE_T); +} + + +// REST IS NOT COPIED FROM CPYTHON Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Fri Apr 2 01:41:43 2010 @@ -10,6 +10,15 @@ #define PYTHON_API_VERSION 1013 #define PYTHON_API_STRING "1013" +int PyArg_Parse(PyObject *, const char *, ...); +int PyArg_ParseTuple(PyObject *, const char *, ...); +int PyArg_VaParse(PyObject *, const char *, va_list); + +int PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, + const char *, char **, ...); +int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, + const char *, char **, va_list); + #define Py_InitModule(name, methods) \ Py_InitModule4(name, methods, (char *)NULL, (PyObject *)NULL, \ PYTHON_API_VERSION) @@ -20,6 +29,8 @@ int PyModule_AddObject(PyObject *m, const char *name, PyObject *o); + + #ifdef __cplusplus } #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 01:41:43 2010 @@ -437,7 +437,7 @@ def test_new_exception(self): skip("not working yet") mod = self.import_extension('foo', [ - ('newexc', 'METHOD_VARARGS', + ('newexc', 'METH_VARARGS', ''' char *name = PyString_AsString(PyTuple_GetItem(args, 0)); return PyExc_NewException(name, PyTuple_GetItem(args, 1), Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Fri Apr 2 01:41:43 2010 @@ -8,9 +8,9 @@ ('oneargint', 'METH_VARARGS', ''' int l; - //if (!PyArg_Parse(args, "i", &l)) { - // return NULL; - //} + if (!PyArg_Parse(args, "i", &l)) { + return NULL; + } return PyInt_FromLong(l); ''' )]) From fijal at codespeak.net Fri Apr 2 01:57:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 01:57:53 +0200 (CEST) Subject: [pypy-svn] r73268 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100401235753.8DC6C282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 01:57:52 2010 New Revision: 73268 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: setup va functions in case we're compiled as well Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 01:57:52 2010 @@ -354,6 +354,15 @@ wrapper.__name__ = "wrapper for %r" % (callable, ) return wrapper +def setup_va_functions(eci): + for name, TP in VA_TP_LIST.iteritems(): + name_no_star = name.strip('*') + ARGS = [lltype.Ptr(va_list)] + func = rffi.llexternal('pypy_va_get_%s' % name_no_star, ARGS, + TP, compilation_info=eci) + globals()['va_get_%s' % name_no_star] = func + + def bootstrap_types(space): from pypy.module.cpyext.pyobject import make_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr, PyPyType_Ready @@ -442,12 +451,7 @@ ll2ctypes.lltype2ctypes(func.get_llhelper(space)), ctypes.c_void_p) - for name, TP in VA_TP_LIST.iteritems(): - name_no_star = name.strip('*') - ARGS = [lltype.Ptr(va_list)] - func = rffi.llexternal('pypy_va_get_%s' % name_no_star, ARGS, - TP, compilation_info=eci) - globals()['va_get_%s' % name_no_star] = func + setup_va_functions(eci) return modulename.new(ext='') @@ -554,6 +558,7 @@ eci = build_eci(False, export_symbols) bootstrap_types(space) + setup_va_functions(eci) # populate static data for name, (type, expr) in GLOBALS.iteritems(): From fijal at codespeak.net Fri Apr 2 02:00:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:00:43 +0200 (CEST) Subject: [pypy-svn] r73269 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402000043.0EAE0282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:00:42 2010 New Revision: 73269 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: Oops, don't check in pdb. raise exception for now Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 02:00:42 2010 @@ -12,8 +12,7 @@ rffi.INT_real, error=-1) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): i = 0 - import pdb - pdb.set_trace() + raise Exception("This is broken so far") while True: c = fmt[i] if c == "\x00": From fijal at codespeak.net Fri Apr 2 02:49:37 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:49:37 +0200 (CEST) Subject: [pypy-svn] r73270 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100402004937.38B8A282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:49:36 2010 New Revision: 73270 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c Log: hack around COpaque object. vgetargs is still unfinished though Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 02:49:36 2010 @@ -44,7 +44,7 @@ class CConfig_constants: _compilation_info_ = CConfig._compilation_info_ -va_list = rffi.COpaque('va_list') +VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list') constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING @@ -354,11 +354,13 @@ wrapper.__name__ = "wrapper for %r" % (callable, ) return wrapper +def process_va_name(name): + return name.replace('*', '_star') + def setup_va_functions(eci): for name, TP in VA_TP_LIST.iteritems(): - name_no_star = name.strip('*') - ARGS = [lltype.Ptr(va_list)] - func = rffi.llexternal('pypy_va_get_%s' % name_no_star, ARGS, + name_no_star = process_va_name(name) + func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P], TP, compilation_info=eci) globals()['va_get_%s' % name_no_star] = func @@ -502,7 +504,7 @@ for name, (typ, expr) in GLOBALS.iteritems(): pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) for name in VA_TP_LIST: - name_no_star = name.strip('*') + name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % (name, name_no_star)) pypy_decls.append(header + ';') Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 02:49:36 2010 @@ -1,6 +1,6 @@ -from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL, va_list,\ - cpython_api_c +from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL, \ + VA_LIST_P, cpython_api_c from pypy.module.cpyext import api from pypy.rpython.lltypesystem import lltype, rffi @@ -8,16 +8,16 @@ def PyArg_Parse(): pass - at cpython_api([PyObject, rffi.CCHARP, lltype.Ptr(va_list), rffi.INT_real], + at cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=-1) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): i = 0 - raise Exception("This is broken so far") while True: c = fmt[i] if c == "\x00": return 0 if c == "i": - pyobj = api.va_get_PyObject(va_list_p); + pyobj = api.va_get_int_star(va_list_p) + # XXX processs.... i += 1 return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c Fri Apr 2 02:49:36 2010 @@ -36,7 +36,7 @@ /* Forward */ -int pypy_vgetargs1(struct _object *, char *, va_list *, int); +int pypy_vgetargs1(struct _object *, char *, char *, int); //static void seterror(int, const char *, int *, const char *, const char *); //static char *convertitem(PyObject *, const char **, va_list *, int, int *, // char *, size_t, PyObject **); @@ -58,7 +58,7 @@ va_list va; va_start(va, format); - retval = pypy_vgetargs1(args, format, &va, FLAG_COMPAT); + retval = pypy_vgetargs1(args, format, (char*)(&va), FLAG_COMPAT); va_end(va); return retval; } From fijal at codespeak.net Fri Apr 2 02:50:05 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:50:05 +0200 (CEST) Subject: [pypy-svn] r73271 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100402005005.0DD9A282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:50:03 2010 New Revision: 73271 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Kill assert for now Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Fri Apr 2 02:50:03 2010 @@ -14,4 +14,4 @@ return PyInt_FromLong(l); ''' )]) - assert mod.oneargint(1) == 1 + mod.oneargint(1) From fijal at codespeak.net Fri Apr 2 02:52:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:52:21 +0200 (CEST) Subject: [pypy-svn] r73272 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402005221.08AD5282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:52:20 2010 New Revision: 73272 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: Don't do anything here Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 02:52:20 2010 @@ -17,7 +17,8 @@ if c == "\x00": return 0 if c == "i": - pyobj = api.va_get_int_star(va_list_p) + #pyobj = api.va_get_int_star(va_list_p) # XXX processs.... + pass i += 1 return 0 From fijal at codespeak.net Fri Apr 2 02:56:58 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:56:58 +0200 (CEST) Subject: [pypy-svn] r73273 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402005658.1B961282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:56:55 2010 New Revision: 73273 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: * a new type * don't set it up twice Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 02:56:55 2010 @@ -261,7 +261,8 @@ cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) VA_TP_LIST = {'int': rffi.INT, - 'PyObject*': PyObject} + 'PyObject*': PyObject, + 'int*': rffi.INTP} def configure_types(): for name, TYPE in rffi_platform.configure(CConfig).iteritems(): @@ -453,8 +454,6 @@ ll2ctypes.lltype2ctypes(func.get_llhelper(space)), ctypes.c_void_p) - setup_va_functions(eci) - return modulename.new(ext='') def generate_macros(export_symbols, rename=True, do_deref=True): From fijal at codespeak.net Fri Apr 2 02:57:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 02:57:14 +0200 (CEST) Subject: [pypy-svn] r73274 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100402005714.4C9F2282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 02:57:12 2010 New Revision: 73274 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: This should be enough to make this test pass Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 02:57:12 2010 @@ -17,8 +17,7 @@ if c == "\x00": return 0 if c == "i": - #pyobj = api.va_get_int_star(va_list_p) - # XXX processs.... - pass + arr = api.va_get_int_star(va_list_p) + arr[0] = space.int_w(space.getitem(w_obj, space.wrap(i))) i += 1 return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Fri Apr 2 02:57:12 2010 @@ -14,4 +14,4 @@ return PyInt_FromLong(l); ''' )]) - mod.oneargint(1) + assert mod.oneargint(1) == 1 From fijal at codespeak.net Fri Apr 2 03:01:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 03:01:55 +0200 (CEST) Subject: [pypy-svn] r73275 - pypy/branch/cpython-extension/pypy/interpreter Message-ID: <20100402010155.36995282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 03:01:54 2010 New Revision: 73275 Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py Log: This is totally needed when running untranslated Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/typedef.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/typedef.py Fri Apr 2 03:01:54 2010 @@ -315,9 +315,9 @@ # - if cls is a string, XXX unused? if cls is None and use_closure: return func - #if hasattr(func, 'im_func'): - # assert func.im_class is cls - # func = func.im_func + if hasattr(func, 'im_func'): + assert func.im_class is cls + func = func.im_func miniglobals = { func.__name__: func, From fijal at codespeak.net Fri Apr 2 03:07:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 03:07:46 +0200 (CEST) Subject: [pypy-svn] r73276 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402010746.D4D6C282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 03:07:45 2010 New Revision: 73276 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: This is needed, but this hack is ugly. I'll kill it once I find something better Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 03:07:45 2010 @@ -454,6 +454,8 @@ ll2ctypes.lltype2ctypes(func.get_llhelper(space)), ctypes.c_void_p) + setup_va_functions(eci) + return modulename.new(ext='') def generate_macros(export_symbols, rename=True, do_deref=True): From fijal at codespeak.net Fri Apr 2 03:09:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 03:09:14 +0200 (CEST) Subject: [pypy-svn] r73277 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402010914.84332282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 03:09:13 2010 New Revision: 73277 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: This should return 1 on success, 0 on failure, pass the first test (yay!) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 03:09:13 2010 @@ -9,15 +9,14 @@ pass @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], - rffi.INT_real, error=-1) + rffi.INT_real, error=0) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): i = 0 while True: c = fmt[i] if c == "\x00": - return 0 + return 1 if c == "i": arr = api.va_get_int_star(va_list_p) arr[0] = space.int_w(space.getitem(w_obj, space.wrap(i))) i += 1 - return 0 From fijal at codespeak.net Fri Apr 2 03:12:25 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 03:12:25 +0200 (CEST) Subject: [pypy-svn] r73278 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100402011225.2EE3F282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 03:12:23 2010 New Revision: 73278 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Pass a bit more tests Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Fri Apr 2 03:12:23 2010 @@ -8,6 +8,10 @@ def PyArg_Parse(): pass + at cpython_api_c() +def PyArg_ParseTuple(): + pass + @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=0) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Fri Apr 2 03:12:23 2010 @@ -15,3 +15,10 @@ ''' )]) assert mod.oneargint(1) == 1 + raises(TypeError, mod.oneargint, None) + try: + mod.oneargint() + except IndexError: + pass + else: + raise Exception("DID NOT RAISE") From benjamin at codespeak.net Fri Apr 2 03:44:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 2 Apr 2010 03:44:25 +0200 (CEST) Subject: [pypy-svn] r73279 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100402014425.46344282B9C@codespeak.net> Author: benjamin Date: Fri Apr 2 03:44:23 2010 New Revision: 73279 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (props changed) Log: set eol From fijal at codespeak.net Fri Apr 2 05:16:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 05:16:18 +0200 (CEST) Subject: [pypy-svn] r73280 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100402031618.A96E3282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 05:16:13 2010 New Revision: 73280 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Define Py_X[INC|DEC]REF Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Fri Apr 2 05:16:13 2010 @@ -39,6 +39,8 @@ #define Py_INCREF(ob) (Py_IncRef(ob)) #define Py_DECREF(ob) (Py_DecRef(ob)) +#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) +#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0) #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) From fijal at codespeak.net Fri Apr 2 06:08:50 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 06:08:50 +0200 (CEST) Subject: [pypy-svn] r73281 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100402040850.AF52A282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 06:08:46 2010 New Revision: 73281 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Avoid double check for nullity Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Fri Apr 2 06:08:46 2010 @@ -39,8 +39,8 @@ #define Py_INCREF(ob) (Py_IncRef(ob)) #define Py_DECREF(ob) (Py_DecRef(ob)) -#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) -#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0) +#define Py_XINCREF(ob) (Py_IncRef(ob)) +#define Py_XDECREF(ob) (Py_DecRef(ob)) #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) From fijal at codespeak.net Fri Apr 2 07:11:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 07:11:30 +0200 (CEST) Subject: [pypy-svn] r73282 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402051130.75948282B9C@codespeak.net> Author: fijal Date: Fri Apr 2 07:11:26 2010 New Revision: 73282 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: mention an issue and kill done Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Fri Apr 2 07:11:26 2010 @@ -3,6 +3,8 @@ - Copy the slots from the base. - Both tasks are necessary to be able to call slots from C code correctly. + - test_typeobject leaks something + - Implement subclassing of types defined in C. - Use a WeakKeyDictionary to count how often a PyObject is allocated for @@ -25,7 +27,3 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. - - - Use the standard CPython attribute names: ob_type, ob_refcnt. - - - ob_type should be a PyTypeObject. From afa at codespeak.net Fri Apr 2 09:51:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 09:51:15 +0200 (CEST) Subject: [pypy-svn] r73284 - pypy/branch/cpython-extension/pypy/interpreter Message-ID: <20100402075115.84A55282B9C@codespeak.net> Author: afa Date: Fri Apr 2 09:51:09 2010 New Revision: 73284 Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py Log: Restore assertion Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/typedef.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/typedef.py Fri Apr 2 09:51:09 2010 @@ -369,7 +369,7 @@ @specialize.arg(0) def make_objclass_getter(tag, func, cls): - if func and not cls and hasattr(func, 'im_func'): + if func and hasattr(func, 'im_func'): assert not cls or cls is func.im_class cls = func.im_class return _make_objclass_getter(cls) From afa at codespeak.net Fri Apr 2 11:12:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 11:12:35 +0200 (CEST) Subject: [pypy-svn] r73285 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100402091235.D9F2B282B9C@codespeak.net> Author: afa Date: Fri Apr 2 11:12:34 2010 New Revision: 73285 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Log: Fix tests on Windows Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 11:12:34 2010 @@ -410,7 +410,7 @@ struct PyPyAPI* pypyAPI = &_pypyAPI; """ % dict(members=structmembers) - functions = generate_decls_and_callbacks(db) + functions = generate_decls_and_callbacks(db, export_symbols) global_objects = [] for name, (type, expr) in GLOBALS.iteritems(): @@ -484,7 +484,7 @@ pypy_macros_h = udir.join('pypy_macros.h') pypy_macros_h.write('\n'.join(pypy_macros)) -def generate_decls_and_callbacks(db): +def generate_decls_and_callbacks(db, export_symbols): # implement function callbacks and generate function decls functions = [] pypy_decls = [] @@ -510,6 +510,8 @@ (name, name_no_star)) pypy_decls.append(header + ';') functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name) + if not we_are_translated(): + export_symbols.append('pypy_va_get_%s' % (name_no_star,)) pypy_decls.append("#endif\n") Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Fri Apr 2 11:12:34 2010 @@ -5,6 +5,11 @@ #endif +#ifndef offsetof +#define offsetof(type, member) ( (int) & ((type*)0) -> member ) +#endif + + typedef struct PyMemberDef { /* Current version, use this */ char *name; From afa at codespeak.net Fri Apr 2 11:19:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 11:19:43 +0200 (CEST) Subject: [pypy-svn] r73286 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402091943.9687F282B9C@codespeak.net> Author: afa Date: Fri Apr 2 11:19:42 2010 New Revision: 73286 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Fix translation. We don't need to export the internal pypy_va_get* function in this case. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 11:19:42 2010 @@ -510,8 +510,7 @@ (name, name_no_star)) pypy_decls.append(header + ';') functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name) - if not we_are_translated(): - export_symbols.append('pypy_va_get_%s' % (name_no_star,)) + export_symbols.append('pypy_va_get_%s' % (name_no_star,)) pypy_decls.append("#endif\n") @@ -558,7 +557,7 @@ generate_macros(export_symbols, rename, False) - generate_decls_and_callbacks(db) + generate_decls_and_callbacks(db, []) eci = build_eci(False, export_symbols) From afa at codespeak.net Fri Apr 2 14:35:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 2 Apr 2010 14:35:42 +0200 (CEST) Subject: [pypy-svn] r73297 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100402123542.4EC25282B9C@codespeak.net> Author: afa Date: Fri Apr 2 14:35:39 2010 New Revision: 73297 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Log: compile the various pypy_va_get_%s with the translated pypy Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 14:35:39 2010 @@ -484,7 +484,7 @@ pypy_macros_h = udir.join('pypy_macros.h') pypy_macros_h.write('\n'.join(pypy_macros)) -def generate_decls_and_callbacks(db, export_symbols): +def generate_decls_and_callbacks(db, export_symbols, api_struct=True): # implement function callbacks and generate function decls functions = [] pypy_decls = [] @@ -496,11 +496,13 @@ arg = arg.replace('@', 'arg%d' % (i,)) args.append(arg) args = ', '.join(args) or "void" - callargs = ', '.join('arg%d' % (i,) for i in range(len(func.argtypes))) header = "%s %s(%s)" % (restype, name, args) pypy_decls.append(header + ";") - body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) - functions.append('%s\n%s\n' % (header, body)) + if api_struct: + callargs = ', '.join('arg%d' % (i,) + for i in range(len(func.argtypes))) + body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) + functions.append('%s\n%s\n' % (header, body)) pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, (typ, expr) in GLOBALS.iteritems(): pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) @@ -518,22 +520,19 @@ pypy_decl_h.write('\n'.join(pypy_decls)) return functions -def build_eci(build_bridge, export_symbols, code=None): +def build_eci(build_bridge, export_symbols, code): # Build code and get pointer to the structure kwds = {} export_symbols_eci = export_symbols[:] if build_bridge: - assert code is not None if sys.platform == "win32": # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] else: kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] - kwds["separate_module_sources"] = [code] export_symbols_eci.append('pypyAPI') else: - assert code is None kwds["includes"] = ['Python.h'] # this is our Python.h eci = ExternalCompilationInfo( @@ -542,6 +541,7 @@ include_dir / "pyerrors.c", include_dir / "modsupport.c", include_dir / "getargs.c"], + separate_module_sources = [code], export_symbols=export_symbols_eci, **kwds ) @@ -557,9 +557,10 @@ generate_macros(export_symbols, rename, False) - generate_decls_and_callbacks(db, []) + functions = generate_decls_and_callbacks(db, [], api_struct=False) + code = "#include \n" + "\n".join(functions) - eci = build_eci(False, export_symbols) + eci = build_eci(False, export_symbols, code) bootstrap_types(space) setup_va_functions(eci) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Fri Apr 2 14:35:39 2010 @@ -4,7 +4,7 @@ extern "C" { #endif - +#include /* For offsetof */ #ifndef offsetof #define offsetof(type, member) ( (int) & ((type*)0) -> member ) #endif From xoraxax at codespeak.net Fri Apr 2 15:02:41 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 15:02:41 +0200 (CEST) Subject: [pypy-svn] r73298 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402130241.D6AE8282B9C@codespeak.net> Author: xoraxax Date: Fri Apr 2 15:02:40 2010 New Revision: 73298 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Readd todo item that was removed in r73282 by fijal. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Fri Apr 2 15:02:40 2010 @@ -27,3 +27,5 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. + + - ob_type should be a PyTypeObject. From xoraxax at codespeak.net Fri Apr 2 17:36:16 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 17:36:16 +0200 (CEST) Subject: [pypy-svn] r73300 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100402153616.6FB5E282B90@codespeak.net> Author: xoraxax Date: Fri Apr 2 17:36:14 2010 New Revision: 73300 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix leaks by calling collect. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 17:36:14 2010 @@ -144,6 +144,8 @@ # check for sane refcnts leaking = False state = self.space.fromcache(State) + import gc + gc.collect() lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) for w_obj, obj in state.py_objects_w2r.iteritems(): From fijal at codespeak.net Fri Apr 2 17:42:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 2 Apr 2010 17:42:55 +0200 (CEST) Subject: [pypy-svn] r73301 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100402154255.5F1C6282B90@codespeak.net> Author: fijal Date: Fri Apr 2 17:42:53 2010 New Revision: 73301 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.h pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Expose a bit more of API from .c files. Requires a bit of tests here and there, will fix this (and implement missing functionality). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 17:42:53 2010 @@ -230,7 +230,7 @@ } for exc_name in ['TypeError', 'ValueError', 'KeyError', 'Exception', - 'BaseException']: + 'BaseException', 'SystemError']: GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.w_' + exc_name) for cpyname, pypyexpr in {"Type": "space.w_type", @@ -254,6 +254,9 @@ PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('struct _object', PyObjectFields, PyObjectStruct) +# a pointer to PyObject +PyObjectP = rffi.CArrayPtr(PyObject) + PyStringObjectStruct = lltype.ForwardReference() PyStringObject = lltype.Ptr(PyStringObjectStruct) PyStringObjectFields = PyObjectFields + \ @@ -540,7 +543,8 @@ separate_module_files=[include_dir / "varargwrapper.c", include_dir / "pyerrors.c", include_dir / "modsupport.c", - include_dir / "getargs.c"], + include_dir / "getargs.c", + include_dir / "stringobject.c"], separate_module_sources = [code], export_symbols=export_symbols_eci, **kwds Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Fri Apr 2 17:42:53 2010 @@ -1,6 +1,15 @@ #ifndef Py_PYTHON_H #define Py_PYTHON_H +// XXX this should be in pyconfig.h + +#define HAVE_LONG_LONG 1 +#define HAVE_STDARG_PROTOTYPES 1 +#define PY_FORMAT_LONG_LONG "ll" +#define PY_LONG_LONG long long +#define SIZEOF_LONG_LONG sizeof(PY_LONG_LONG) +#define PY_FORMAT_SIZE_T "z" + /* Compat stuff */ #ifndef _WIN32 # include @@ -18,6 +27,16 @@ #endif #define Py_ssize_t long +/* Convert a possibly signed character to a nonnegative int */ +/* XXX This assumes characters are 8 bits wide */ +#ifdef __CHAR_UNSIGNED__ +#define Py_CHARMASK(c) (c) +#else +#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) +#endif + +#define Py_MEMCPY memcpy + #include #include "patchlevel.h" @@ -28,6 +47,8 @@ #include #include #include +#include +#include #include "boolobject.h" #include "floatobject.h" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c Fri Apr 2 17:42:53 2010 @@ -1,6 +1,5 @@ #include -#if 0 -unsupported functions in it + int PyModule_AddObject(PyObject *m, const char *name, PyObject *o) { @@ -29,4 +28,4 @@ Py_DECREF(o); return 0; } -#endif + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c Fri Apr 2 17:42:53 2010 @@ -1,5 +1,25 @@ #include #include + +PyObject * +PyErr_Format(PyObject *exception, const char *format, ...) +{ + va_list vargs; + PyObject* string; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + + string = PyString_FromFormatV(format, vargs); + PyErr_SetObject(exception, string); + Py_XDECREF(string); + va_end(vargs); + return NULL; +} + #if 0 depends on unavailable functions @@ -52,4 +72,6 @@ Py_XDECREF(modulename); return result; } + + #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.h Fri Apr 2 17:42:53 2010 @@ -8,6 +8,7 @@ #endif PyObject *PyErr_NewException(char *name, PyObject *base, PyObject *dict); +PyObject *PyErr_Format(PyObject *exception, const char *format, ...); #ifdef __cplusplus } Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c Fri Apr 2 17:42:53 2010 @@ -0,0 +1,233 @@ + +#include "Python.h" + +PyObject * +PyString_FromFormatV(const char *format, va_list vargs) +{ + va_list count; + Py_ssize_t n = 0; + const char* f; + char *s; + PyObject* string; + +#ifdef VA_LIST_IS_ARRAY + Py_MEMCPY(count, vargs, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(count, vargs); +#else + count = vargs; +#endif +#endif + /* step 1: figure out how large a buffer we need */ + for (f = format; *f; f++) { + if (*f == '%') { +#ifdef HAVE_LONG_LONG + int longlongflag = 0; +#endif + const char* p = f; + while (*++f && *f != '%' && !isalpha(Py_CHARMASK(*f))) + ; + + /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since + * they don't affect the amount of space we reserve. + */ + if (*f == 'l') { + if (f[1] == 'd' || f[1] == 'u') { + ++f; + } +#ifdef HAVE_LONG_LONG + else if (f[1] == 'l' && + (f[2] == 'd' || f[2] == 'u')) { + longlongflag = 1; + f += 2; + } +#endif + } + else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + ++f; + } + + switch (*f) { + case 'c': + (void)va_arg(count, int); + /* fall through... */ + case '%': + n++; + break; + case 'd': case 'u': case 'i': case 'x': + (void) va_arg(count, int); +#ifdef HAVE_LONG_LONG + /* Need at most + ceil(log10(256)*SIZEOF_LONG_LONG) digits, + plus 1 for the sign. 53/22 is an upper + bound for log10(256). */ + if (longlongflag) + n += 2 + (SIZEOF_LONG_LONG*53-1) / 22; + else +#endif + /* 20 bytes is enough to hold a 64-bit + integer. Decimal takes the most + space. This isn't enough for + octal. */ + n += 20; + + break; + case 's': + s = va_arg(count, char*); + n += strlen(s); + break; + case 'p': + (void) va_arg(count, int); + /* maximum 64-bit pointer representation: + * 0xffffffffffffffff + * so 19 characters is enough. + * XXX I count 18 -- what's the extra for? + */ + n += 19; + break; + default: + /* if we stumble upon an unknown + formatting code, copy the rest of + the format string to the output + string. (we cannot just skip the + code, since there's no way to know + what's in the argument list) */ + n += strlen(p); + goto expand; + } + } else + n++; + } + expand: + /* step 2: fill the buffer */ + /* Since we've analyzed how much space we need for the worst case, + use sprintf directly instead of the slower PyOS_snprintf. */ + string = PyString_FromStringAndSize(NULL, n); + if (!string) + return NULL; + + s = PyString_AsString(string); + + for (f = format; *f; f++) { + if (*f == '%') { + const char* p = f++; + Py_ssize_t i; + int longflag = 0; +#ifdef HAVE_LONG_LONG + int longlongflag = 0; +#endif + int size_tflag = 0; + /* parse the width.precision part (we're only + interested in the precision value, if any) */ + n = 0; + while (isdigit(Py_CHARMASK(*f))) + n = (n*10) + *f++ - '0'; + if (*f == '.') { + f++; + n = 0; + while (isdigit(Py_CHARMASK(*f))) + n = (n*10) + *f++ - '0'; + } + while (*f && *f != '%' && !isalpha(Py_CHARMASK(*f))) + f++; + /* Handle %ld, %lu, %lld and %llu. */ + if (*f == 'l') { + if (f[1] == 'd' || f[1] == 'u') { + longflag = 1; + ++f; + } +#ifdef HAVE_LONG_LONG + else if (f[1] == 'l' && + (f[2] == 'd' || f[2] == 'u')) { + longlongflag = 1; + f += 2; + } +#endif + } + /* handle the size_t flag. */ + else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + size_tflag = 1; + ++f; + } + + switch (*f) { + case 'c': + *s++ = va_arg(vargs, int); + break; + case 'd': + if (longflag) + sprintf(s, "%ld", va_arg(vargs, long)); +#ifdef HAVE_LONG_LONG + else if (longlongflag) + sprintf(s, "%" PY_FORMAT_LONG_LONG "d", + va_arg(vargs, PY_LONG_LONG)); +#endif + else if (size_tflag) + sprintf(s, "%" PY_FORMAT_SIZE_T "d", + va_arg(vargs, Py_ssize_t)); + else + sprintf(s, "%d", va_arg(vargs, int)); + s += strlen(s); + break; + case 'u': + if (longflag) + sprintf(s, "%lu", + va_arg(vargs, unsigned long)); +#ifdef HAVE_LONG_LONG + else if (longlongflag) + sprintf(s, "%" PY_FORMAT_LONG_LONG "u", + va_arg(vargs, PY_LONG_LONG)); +#endif + else if (size_tflag) + sprintf(s, "%" PY_FORMAT_SIZE_T "u", + va_arg(vargs, size_t)); + else + sprintf(s, "%u", + va_arg(vargs, unsigned int)); + s += strlen(s); + break; + case 'i': + sprintf(s, "%i", va_arg(vargs, int)); + s += strlen(s); + break; + case 'x': + sprintf(s, "%x", va_arg(vargs, int)); + s += strlen(s); + break; + case 's': + p = va_arg(vargs, char*); + i = strlen(p); + if (n > 0 && i > n) + i = n; + Py_MEMCPY(s, p, i); + s += i; + break; + case 'p': + sprintf(s, "%p", va_arg(vargs, void*)); + /* %p is ill-defined: ensure leading 0x. */ + if (s[1] == 'X') + s[1] = 'x'; + else if (s[1] != 'x') { + memmove(s+2, s, strlen(s)+1); + s[0] = '0'; + s[1] = 'x'; + } + s += strlen(s); + break; + case '%': + *s++ = '%'; + break; + default: + strcpy(s, p); + s += strlen(s); + goto end; + } + } else + *s++ = *f; + } + + end: + _PyString_Resize(&string, s - PyString_AS_STRING(string)); + return string; +} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Fri Apr 2 17:42:53 2010 @@ -7,12 +7,17 @@ extern "C" { #endif +#define PyString_AS_STRING(op) PyString_AsString(op) + typedef struct { PyObject_HEAD char* buffer; Py_ssize_t size; } PyStringObject; +PyObject * +PyString_FromFormatV(const char *format, va_list vargs); + #ifdef __cplusplus } #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Fri Apr 2 17:42:53 2010 @@ -91,3 +91,13 @@ else: PyErr_BadInternalCall(space) + at cpython_api([PyObject], rffi.CCHARP, error=0) +def PyModule_GetName(space, module): + """ + + + + Return module's __name__ value. If the module does not provide one, + or if it is not a string, SystemError is raised and NULL is returned.""" + raise NotImplementedError + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Fri Apr 2 17:42:53 2010 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, - CANNOT_FAIL, build_type_checkers) + CANNOT_FAIL, build_type_checkers, + PyObjectP) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref @@ -48,3 +49,19 @@ else: w_obj = from_ref(space, ref) return space.int_w(space.len(w_obj)) + + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) +def _PyString_Resize(space, string, newsize): + """A way to resize a string object even though it is "immutable". Only use this to + build up a brand new string object; don't use this if the string may already be + known in other parts of the code. It is an error to call this function if the + refcount on the input string object is not one. Pass the address of an existing + string object as an lvalue (it may be written into), and the new size desired. + On success, *string holds the resized string object and 0 is returned; + the address in *string may differ from its input value. If the reallocation + fails, the original string object at *string is deallocated, *string is + set to NULL, a memory exception is set, and -1 is returned. + + This function used an int type for newsize. This might + require changes in your code for properly supporting 64-bit systems.""" + raise NotImplementedError Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py Fri Apr 2 17:42:53 2010 @@ -19,6 +19,7 @@ "PyVarObject*": "PyObject", "const char*": "rffi.CCHARP", "PyObject*": "PyObject", + "PyObject**": "PyObjectP", "char*": "rffi.CCHARP", "PyMethodDef*": "PyMethodDef", "Py_ssize_t": "Py_ssize_t", Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 2 17:42:53 2010 @@ -4033,16 +4033,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.CCHARP) -def PyModule_GetName(space, module): - """ - - - - Return module's __name__ value. If the module does not provide one, - or if it is not a string, SystemError is raised and NULL is returned.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) def PyModule_GetFilename(space, module): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Fri Apr 2 17:42:53 2010 @@ -44,8 +44,7 @@ ("test_is_string", "METH_VARARGS", """ return PyBool_FromLong(PyString_Check(PyTuple_GetItem(args, 0))); - """), - ]) + """)]) assert module.get_hello1() == 'Hello world' assert module.get_hello2() == 'Hello world' assert module.test_Size() @@ -96,3 +95,16 @@ ]) s = module.getstring() assert s == 'test' + + def test_format_v(self): + skip("unsupported yet, think how to fak va_list") + module = self.import_extension('foo', [ + ("test_string_format_v", "METH_VARARGS", + ''' + return PyString_FromFormatV("bla %d ble %s", + PyInt_AsLong(PyTuple_GetItem(args, 0)), + PyString_AsString(PyTuple_GetItem(args, 1))); + ''' + ) + ]) + pass From exarkun at codespeak.net Fri Apr 2 18:38:23 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 2 Apr 2010 18:38:23 +0200 (CEST) Subject: [pypy-svn] r73302 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100402163823.8D673282B90@codespeak.net> Author: exarkun Date: Fri Apr 2 18:38:21 2010 New Revision: 73302 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Add PyDoc_STRVAR and related macros; also extend PyMethodDef to take a docstring Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Fri Apr 2 18:38:21 2010 @@ -9,6 +9,7 @@ #define PY_LONG_LONG long long #define SIZEOF_LONG_LONG sizeof(PY_LONG_LONG) #define PY_FORMAT_SIZE_T "z" +#define WITH_DOC_STRINGS /* Compat stuff */ #ifndef _WIN32 @@ -69,4 +70,13 @@ #include +/* Define macros for inline documentation. */ +#define PyDoc_VAR(name) static char name[] +#define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str) +#ifdef WITH_DOC_STRINGS +#define PyDoc_STR(str) str +#else +#define PyDoc_STR(str) "" +#endif + #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Fri Apr 2 18:38:21 2010 @@ -15,9 +15,10 @@ class W_PyCFunctionObject(Wrappable): - def __init__(self, space, ml, w_self): + def __init__(self, space, ml, w_self, doc=None): self.ml = ml self.w_self = w_self + self.doc = doc def call(self, space, w_self, args_tuple): # Call the C function @@ -117,6 +118,7 @@ W_PyCFunctionObject.typedef = TypeDef( 'builtin_function_or_method', __call__ = interp2app(cfunction_descr_call), + __doc__ = interp_attrproperty('doc', cls=W_PyCFunctionObject), ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Fri Apr 2 18:38:21 2010 @@ -3,7 +3,7 @@ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module -from pypy.module.cpyext.methodobject import PyCFunction_NewEx, PyDescr_NewMethod +from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -14,6 +14,7 @@ [('ml_name', rffi.CCHARP), ('ml_meth', PyCFunction), ('ml_flags', rffi.INT_real), + ('ml_doc', rffi.CCHARP), ]) def PyImport_AddModule(space, name): @@ -51,11 +52,16 @@ methodname = rffi.charp2str(method.c_ml_name) flags = rffi.cast(lltype.Signed, method.c_ml_flags) + if method.c_ml_doc: + doc = rffi.charp2str(method.c_ml_doc) + else: + doc = None + if not pto: if flags & METH_CLASS or flags & METH_STATIC: raise OperationError(space.w_ValueError, space.wrap("module functions cannot set METH_CLASS or METH_STATIC")) - w_obj = PyCFunction_NewEx(space, method, w_self) + w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, doc)) else: if methodname in dict_w and not (flags & METH_COEXIST): continue @@ -71,6 +77,7 @@ #w_obj = PyStaticMethod_New(space, w_func) else: w_obj = PyDescr_NewMethod(space, pto, method) + dict_w[methodname] = w_obj Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 18:38:21 2010 @@ -195,6 +195,29 @@ assert module.return_pi is not None assert module.return_pi() == 3.14 + + def test_export_docstring(self): + import sys + init = """ + if (Py_IsInitialized()) + Py_InitModule("foo", methods); + """ + body = """ + PyDoc_STRVAR(foo_pi_doc, "Return pi."); + PyObject* foo_pi(PyObject* self, PyObject *args) + { + return PyFloat_FromDouble(3.14); + } + static PyMethodDef methods[] ={ + { "return_pi", foo_pi, METH_NOARGS, foo_pi_doc }, + { NULL } + }; + """ + module = self.import_module(name='foo', init=init, body=body) + doc = module.return_pi.__doc__ + assert doc == "Return pi." + + def test_InitModule4(self): init = """ PyObject *cookie = PyFloat_FromDouble(3.14); From exarkun at codespeak.net Fri Apr 2 19:51:34 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 2 Apr 2010 19:51:34 +0200 (CEST) Subject: [pypy-svn] r73303 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100402175134.DD0C2282B90@codespeak.net> Author: exarkun Date: Fri Apr 2 19:51:31 2010 New Revision: 73303 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: Attempt to add PyErr_SetFromErrno; the test currently fails because it gets the wrong errno back from the OSError raised Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 19:51:31 2010 @@ -230,7 +230,7 @@ } for exc_name in ['TypeError', 'ValueError', 'KeyError', 'Exception', - 'BaseException', 'SystemError']: + 'BaseException', 'SystemError', 'OSError']: GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.w_' + exc_name) for cpyname, pypyexpr in {"Type": "space.w_type", Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Fri Apr 2 19:51:31 2010 @@ -1,8 +1,11 @@ +import os + from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, make_ref, register_container from pypy.module.cpyext.state import State +from pypy.rlib.rposix import get_errno @cpython_api([PyObject, PyObject], lltype.Void) def PyErr_SetObject(space, w_type, w_value): @@ -16,6 +19,28 @@ message = rffi.charp2str(message_ptr) PyErr_SetObject(space, w_type, space.wrap(message)) + + at cpython_api([PyObject], PyObject) +def PyErr_SetFromErrno(space, w_type): + """ + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable errno. It constructs a + tuple object whose first item is the integer errno value and whose + second item is the corresponding error message (gotten from strerror()), + and then calls PyErr_SetObject(type, object). On Unix, when the + errno value is EINTR, indicating an interrupted system call, + this calls PyErr_CheckSignals(), and if that set the error indicator, + leaves it set to that. The function always returns NULL, so a wrapper + function around a system call can write return PyErr_SetFromErrno(type); + when the system call returns an error. + Return value: always NULL.""" + # XXX Doesn't actually do anything with PyErr_CheckSignals. + errno = get_errno() + errno_w = space.wrap(errno) + message_w = space.wrap(os.strerror(errno)) + PyErr_SetObject(space, w_type, errno_w, message_w) + + @cpython_api([], PyObject, borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 2 19:51:31 2010 @@ -1721,25 +1721,6 @@ Return value: always NULL.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyErr_SetFromErrno(space, type): - """ - - - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable errno. It constructs a - tuple object whose first item is the integer errno value and whose - second item is the corresponding error message (gotten from strerror()), - and then calls PyErr_SetObject(type, object). On Unix, when the - errno value is EINTR, indicating an interrupted system call, - this calls PyErr_CheckSignals(), and if that set the error indicator, - leaves it set to that. The function always returns NULL, so a wrapper - function around a system call can write return PyErr_SetFromErrno(type); - when the system call returns an error. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyErr_SetFromErrnoWithFilename(space, type, filename): """Similar to PyErr_SetFromErrno(), with the additional behavior that if Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Fri Apr 2 19:51:31 2010 @@ -52,6 +52,8 @@ api.PyErr_Clear() + + class AppTestFetch(AppTestCpythonExtensionBase): def test_occurred(self): module = self.import_extension('foo', [ @@ -65,3 +67,24 @@ ), ]) module.check_error() + + + def test_SetFromErrno(self): + import errno + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + int close(int); + close(-1); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + '''), + ]) + try: + module.set_from_errno() + except OSError, e: + assert e.errno == errno.EBADF + assert e.message == os.strerror(errno.EBADF) + + From exarkun at codespeak.net Fri Apr 2 20:45:28 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 2 Apr 2010 20:45:28 +0200 (CEST) Subject: [pypy-svn] r73304 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100402184528.2BB34282B90@codespeak.net> Author: exarkun Date: Fri Apr 2 20:45:26 2010 New Revision: 73304 Added: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Implement PyLong_FromLong Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Fri Apr 2 20:45:26 2010 @@ -42,6 +42,7 @@ import pypy.module.cpyext.tupleobject import pypy.module.cpyext.dictobject import pypy.module.cpyext.intobject +import pypy.module.cpyext.longobject import pypy.module.cpyext.listobject import pypy.module.cpyext.sequence import pypy.module.cpyext.eval Added: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Fri Apr 2 20:45:26 2010 @@ -0,0 +1,11 @@ + +from pypy.rpython.lltypesystem import lltype +from pypy.module.cpyext.api import cpython_api, PyObject + + + at cpython_api([lltype.Signed], PyObject) +def PyLong_FromLong(space, val): + """Return a new PyLongObject object from v, or NULL on failure.""" + return space.wrap(val) + + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 2 20:45:26 2010 @@ -3549,11 +3549,6 @@ """ raise NotImplementedError - at cpython_api([{long}], PyObject) -def PyLong_FromLong(space, v): - """Return a new PyLongObject object from v, or NULL on failure.""" - raise NotImplementedError - @cpython_api([{unsigned long}], PyObject) def PyLong_FromUnsignedLong(space, v): """Return a new PyLongObject object from a C unsigned long, or Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Fri Apr 2 20:45:26 2010 @@ -0,0 +1,17 @@ + +import sys + +from pypy.objspace.std.intobject import W_IntObject +from pypy.objspace.std.longobject import W_LongObject +from pypy.module.cpyext.test.test_api import BaseApiTest + + +class TestLongObject(BaseApiTest): + def test_FromLong(self, space, api): + value = api.PyLong_FromLong(3) + assert isinstance(value, W_IntObject) + assert space.unwrap(value) == 3 + + value = api.PyLong_FromLong(sys.maxint + 1) + assert isinstance(value, W_LongObject) + assert space.unwrap(value) == sys.maxint + 1 From exarkun at codespeak.net Fri Apr 2 21:05:23 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 2 Apr 2010 21:05:23 +0200 (CEST) Subject: [pypy-svn] r73306 - in pypy/branch/cpython-extension/pypy/module/cpyext: include test Message-ID: <20100402190523.451E0282B90@codespeak.net> Author: exarkun Date: Fri Apr 2 21:04:38 2010 New Revision: 73306 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/modinit.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Add a PyMODINIT_FUNC macro, plus a skipped test that fails for reasons probably unrelated to the implementation of PyMODINIT_FUNC Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Fri Apr 2 21:04:38 2010 @@ -29,6 +29,10 @@ int PyModule_AddObject(PyObject *m, const char *name, PyObject *o); +/* + * This is from pyport.h. Perhaps it belongs elsewhere. + */ +#define PyMODINIT_FUNC void #ifdef __cplusplus Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/modinit.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/modinit.c Fri Apr 2 21:04:38 2010 @@ -0,0 +1,16 @@ + +/* + * Trivial module which uses the PyMODINIT_FUNC macro. + */ + +#include + +static PyMethodDef methods[] = { + { NULL } +}; + +PyMODINIT_FUNC +initmodinit(void) { + Py_InitModule3("modinit", methods, ""); +} + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 21:04:38 2010 @@ -245,6 +245,17 @@ assert module.__doc__ == "docstring" assert module.return_cookie() == 3.14 + + def test_modinit_func(self): + """ + A module can use the PyMODINIT_FUNC macro to declare or define its + module initializer function. + """ + skip("This leaks references for some reason") + module = self.import_module(name='modinit') + assert module.__name__ == 'modinit' + + def test_export_function2(self): import sys init = """ From xoraxax at codespeak.net Fri Apr 2 21:15:04 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 21:15:04 +0200 (CEST) Subject: [pypy-svn] r73307 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100402191504.BE243282B90@codespeak.net> Author: xoraxax Date: Fri Apr 2 21:14:47 2010 New Revision: 73307 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Reworked complete object model, added PyUnicodeType to globals, extended foo.c to contain a subtype of the unicode type, began to implement slot functions. The new object model requires a word on every W_Root object but I fear that is not avoidable. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 2 21:14:47 2010 @@ -49,7 +49,7 @@ constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS -Py_TPFLAGS_HEAPTYPE +Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS """.split() for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) @@ -238,6 +238,7 @@ "Dict": "space.w_dict", "Tuple": "space.w_tuple", "List": "space.w_list", + "Unicode": "space.w_unicode", }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Fri Apr 2 21:14:47 2010 @@ -63,6 +63,7 @@ #include "tupleobject.h" #include "dictobject.h" #include "intobject.h" +#include "unicodeobject.h" #include "eval.h" // XXX This shouldn't be included here Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Fri Apr 2 21:14:47 2010 @@ -0,0 +1,3 @@ +typedef struct { + PyObject_HEAD +} PyUnicodeObject; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Fri Apr 2 21:14:47 2010 @@ -3,22 +3,16 @@ from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject, W_PyCObject,\ - W_PyCObjectDual +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject from pypy.objspace.std.objectobject import W_ObjectObject +from pypy.objspace.std.typeobject import W_TypeObject import pypy.module.__builtin__.operation as operation @cpython_api([PyObject], PyObject) def _PyObject_New(space, w_type): if isinstance(w_type, W_PyCTypeObject): - w_pycobj = space.allocate_instance(W_PyCObject, w_type) - w_pycobj.__init__(space) - w_pycobjd = space.allocate_instance(W_PyCObjectDual, w_type) - w_pycobjd.__init__(space) - w_pycobjd.set_pycobject(w_pycobj) - w_pycobj.set_dual(w_pycobjd) - Py_IncRef(space, w_pycobj) - return w_pycobj + w_obj = space.allocate_instance(W_ObjectObject, w_type) + return w_obj assert False, "Please add more cases in get_cls_for_type_object!" @cpython_api([rffi.VOIDP_real], lltype.Void) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 2 21:14:47 2010 @@ -2,7 +2,8 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject, PyStringObject, ADDR +from pypy.module.cpyext.api import cpython_api, PyObject, PyStringObject, ADDR,\ + Py_TPFLAGS_HEAPTYPE from pypy.module.cpyext.state import State from pypy.objspace.std.stringobject import W_StringObject from pypy.rlib.objectmodel import we_are_translated @@ -30,32 +31,40 @@ def make_ref(space, w_obj, borrowed=False, steal=False): from pypy.module.cpyext.typeobject import allocate_type_obj,\ - W_PyCTypeObject, W_PyCObject, W_PyCObjectDual + W_PyCTypeObject, PyOLifeline + from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) state = space.fromcache(State) - if isinstance(w_obj, W_PyCObject): - w_obj = w_obj.w_dual py_obj = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not py_obj: + assert not steal w_type = space.type(w_obj) - if space.is_w(w_type, space.w_type): + if space.is_w(w_type, space.w_type) or space.is_w(w_type, + space.gettypeobject(W_PyCTypeObject.typedef)): pto = allocate_type_obj(space, w_obj) py_obj = rffi.cast(PyObject, pto) # c_ob_type and c_ob_refcnt are set by allocate_type_obj - elif isinstance(w_obj, W_PyCObject): - w_type = space.type(w_obj) - assert isinstance(w_type, W_PyCTypeObject) - pto = w_type.pto - # Don't increase refcount for non-heaptypes - # Py_IncRef(space, pto) - basicsize = pto.c_tp_basicsize - py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, - flavor="raw", zero=True) - py_obj = rffi.cast(PyObject, py_obj_pad) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyObject, pto) + elif isinstance(w_type, W_PyCTypeObject): + lifeline = w_obj.get_pyolifeline() + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + Py_IncRef(space, py_obj) + else: + w_type_pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, w_type_pyo) + # Don't increase refcount for non-heaptypes + if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, pto) + basicsize = pto.c_tp_basicsize + py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, + flavor="raw", zero=True) + py_obj = rffi.cast(PyObject, py_obj_pad) + py_obj.c_ob_refcnt = 1 + py_obj.c_ob_type = rffi.cast(PyObject, pto) + w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) elif isinstance(w_obj, W_StringObject): py_obj_str = lltype.malloc(PyStringObject.TO, flavor='raw', zero=True) py_obj_str.c_size = len(space.str_w(w_obj)) @@ -99,7 +108,6 @@ def from_ref(space, ref): - from pypy.module.cpyext.typeobject import W_PyCObjectDual, W_PyCObject assert lltype.typeOf(ref) == PyObject if not ref: return None @@ -107,19 +115,6 @@ ptr = rffi.cast(ADDR, ref) try: w_obj = state.py_objects_r2w[ptr] - if isinstance(w_obj, W_PyCObjectDual): - w_obj_wr = w_obj.w_pycobject - w_obj_or_None = w_obj_wr() - if w_obj_or_None is None: - # resurrect new PyCObject - Py_IncRef(space, ref) - w_obj_new = space.allocate_instance(W_PyCObject, space.type(w_obj)) - w_obj_new.__init__(space) - w_obj_new.set_dual(w_obj) - w_obj.set_pycobject(w_obj_new) - w_obj = w_obj_new - else: - w_obj = w_obj_or_None except KeyError: ref_type = ref.c_ob_type if ref != ref_type and space.is_w(from_ref(space, ref_type), space.w_str): @@ -138,7 +133,7 @@ if not obj: return - from pypy.module.cpyext.typeobject import string_dealloc + from pypy.module.cpyext.typeobject import string_dealloc, W_PyCTypeObject obj.c_ob_refcnt -= 1 if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) @@ -153,9 +148,12 @@ else: w_obj = state.py_objects_r2w[ptr] del state.py_objects_r2w[ptr] - _Py_Dealloc(space, obj) + w_type = space.type(w_obj) + w_typetype = space.type(w_type) + if not space.is_w(w_typetype, space.gettypeobject(W_PyCTypeObject.typedef)): + _Py_Dealloc(space, obj) del state.py_objects_w2r[w_obj] - if ptr in state.borrow_mapping: + if ptr in state.borrow_mapping: # move to lifeline __del__ for containee in state.borrow_mapping[ptr]: w_containee = state.py_objects_r2w.get(containee, None) if w_containee is not None: @@ -188,7 +186,7 @@ from pypy.module.cpyext.api import generic_cpy_call_dont_decref pto = obj.c_ob_type pto = rffi.cast(PyTypeObjectPtr, pto) - #print >>sys.stderr, "Calling dealloc slot of", obj, \ + #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ # "'s type which is", rffi.charp2str(pto.c_tp_name) generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Fri Apr 2 21:14:47 2010 @@ -4,7 +4,8 @@ from pypy.module.cpyext.api import generic_cpy_call, cpython_api, \ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ - ternaryfunc + ternaryfunc, PyTypeObjectPtr +from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.unroll import unrolling_iterable @@ -32,12 +33,25 @@ func_target = rffi.cast(ternaryfunc, func) return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) + at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) +def slot_tp_new(space, type, w_args, w_kwds): + from pypy.module.cpyext.tupleobject import PyTuple_Check + pyo = rffi.cast(PyObject, type) + w_type = from_ref(space, pyo) + w_func = space.getattr(w_type, space.wrap("__new__")) + assert PyTuple_Check(space, w_args) + args_w = space.listview(w_args)[:] + args_w.insert(0, w_type) + w_args_new = space.newtuple(args_w) + return space.call(w_func, w_args_new, w_kwds) + PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c def FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, FLAGS): wrapper = globals().get(WRAPPER, None) + function = globals().get(FUNCTION, None) slotname = ("c_" + SLOT).split(".") assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS if FLAGS: @@ -46,7 +60,7 @@ else: wrapper1 = wrapper wrapper2 = None - return (NAME, slotname, FUNCTION, wrapper1, wrapper2, DOC) + return (NAME, slotname, function, wrapper1, wrapper2, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Fri Apr 2 21:14:47 2010 @@ -136,6 +136,81 @@ foo_getseters, /*tp_getset*/ }; +typedef struct { + PyUnicodeObject HEAD; +} FuuObject; + +static PyObject * +Fuu_escape(PyTypeObject* type, PyObject *args) +{ + Py_RETURN_TRUE; +} + + +static PyMethodDef Fuu_methods[] = { + {"escape", (PyCFunction) Fuu_escape, METH_VARARGS, NULL}, + {NULL} /* Sentinel */ +}; + +PyTypeObject FuuType = { + PyObject_HEAD_INIT(NULL) + 0, + "foo.fuu", + sizeof(FuuObject), + 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ + 0, /*tp_doc*/ + + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + Fuu_methods,/*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + 0, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + 0, /*tp_new*/ + 0, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + /* foo functions */ @@ -167,13 +242,23 @@ PyObject *m, *d; Py_TYPE(&footype) = &PyType_Type; + + /* Workaround for quirk in Visual Studio, see + */ + FuuType.tp_base = &PyUnicode_Type; + if (PyType_Ready(&footype) < 0) return; + if (PyType_Ready(&FuuType) < 0) + return; m = Py_InitModule("foo", foo_functions); if (m == NULL) return; d = PyModule_GetDict(m); - if (d) - PyDict_SetItemString(d, "fooType", (PyObject *)&footype); + if (d) { + if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + return; + PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); + } /* No need to check the error here, the caller will do that */ } Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 21:14:47 2010 @@ -157,6 +157,22 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) + lifeline = w_obj.get_pyolifeline() + if lifeline is not None: + refcnt = lifeline.pyo.c_ob_refcnt + if refcnt > 0: + print >>sys.stderr, "\tThe object also held by C code." + else: + referrers_repr = [] + for o in gc.get_referrers(w_obj): + try: + repr_str = repr(o) + except TypeError, e: + repr_str = "%s (type of o is %s)" % (str(e), type(o)) + referrers_repr.append(repr_str) + referrers = ", ".join(referrers_repr) + print >>sys.stderr, "\tThe object is referenced by these objects:", \ + referrers for w_obj in lost_objects_w: print >>sys.stderr, "Lost object %r" % (w_obj, ) leaking = True Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Fri Apr 2 21:14:47 2010 @@ -36,3 +36,12 @@ raises(TypeError, "obj.int_member_readonly = 42") raises(SystemError, "obj.broken_member") raises(SystemError, "obj.broken_member = 42") + a = module.fooType + assert "cannot create" in raises(TypeError, "a()").value.message + skip("In Progress") + class bar(module.fooType): + pass + fuu = module.FuuType + fuu_inst = fuu(u"abc") + assert "cannot create" in raises(TypeError, "bar()").value.message + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 2 21:14:47 2010 @@ -3,27 +3,31 @@ from weakref import ref from pypy.rpython.lltypesystem import rffi, lltype +from pypy.tool.pairtype import extendabletype from pypy.rpython.annlowlevel import llhelper -from pypy.interpreter.gateway import ObjSpace, W_Root +from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import Wrappable -from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE +from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type +from pypy.objspace.std.typetype import _precheck_for_new from pypy.objspace.std.objectobject import W_ObjectObject from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ + Py_TPFLAGS_HAVE_CLASS from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.state import State from pypy.module.cpyext.methodobject import PyDescr_NewWrapper -from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef +from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ PyGetSetDef, PyMemberDef from pypy.module.cpyext.slotdefs import slotdefs +from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.rstring import rsplit @@ -35,11 +39,11 @@ if doc: doc = rffi.charp2str(getset.c_doc) if getset.c_get: - get = W_PyCObject.getter + get = GettersAndSetters.getter.im_func if getset.c_set: - set = W_PyCObject.setter + set = GettersAndSetters.setter.im_func GetSetProperty.__init__(self, get, set, None, doc, - cls=W_PyCObject, use_closure=True, + cls=None, use_closure=True, # XXX cls? tag="cpyext_1") def PyDescr_NewGetSet(space, getset, pto): @@ -53,11 +57,11 @@ doc = set = None if doc: doc = rffi.charp2str(getset.c_doc) - get = W_PyCObject.member_getter + get = GettersAndSetters.member_getter.im_func if not (flags & structmemberdefs.READONLY): - set = W_PyCObject.member_setter + set = GettersAndSetters.member_setter.im_func GetSetProperty.__init__(self, get, set, None, doc, - cls=W_PyCObject, use_closure=True, + cls=None, use_closure=True, # XXX cls? tag="cpyext_2") def convert_getset_defs(space, dict_w, getsets, pto): @@ -88,13 +92,38 @@ dict_w[name] = w_descr i += 1 +def update_all_slots(space, w_obj, pto): + # XXX fill slots in pto + state = space.fromcache(State) + for method_name, slot_name, slot_func, _, _, doc in slotdefs: + w_descr = space.lookup(w_obj, method_name) + if w_descr is None: + # XXX special case iternext + continue + if slot_func is None: + os.write(2, method_name + " defined by the type but no slot function defined!\n") + continue + if method_name == "__new__" and "bar" in repr(w_obj): + import pdb; pdb.set_trace() + slot_func_helper = llhelper(slot_func.api_func.functype, + slot_func.api_func.get_wrapper(space)) + # XXX special case wrapper-functions and use a "specific" slot func, + # XXX special case tp_new + if len(slot_name) == 1: + setattr(pto, slot_name[0], slot_func_helper) + else: + assert len(slot_name) == 2 + struct = getattr(pto, slot_name[0]) + if not struct: + continue + setattr(struct, slot_name[1], slot_func_helper) + def add_operators(space, dict_w, pto): # XXX support PyObject_HashNotImplemented state = space.fromcache(State) for method_name, slot_name, _, wrapper_func, wrapper_func_kwds, doc in slotdefs: if method_name in dict_w: continue - # XXX is this rpython? if len(slot_name) == 1: func = getattr(pto, slot_name[0]) else: @@ -107,16 +136,26 @@ if not func: continue if wrapper_func is None and wrapper_func_kwds is None: - os.write(2, method_name + " used by the type but no wrapper function defined!") + os.write(2, method_name + " used by the type but no wrapper function defined!\n") continue dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, func_voidp) +def inherit_special(space, pto, base_pto): + # XXX copy basicsize and flags in a magical way + flags = rffi.cast(lltype.Signed, pto.c_tp_flags) + if flags & Py_TPFLAGS_HAVE_CLASS: + base_object_pyo = make_ref(space, space.w_object, steal=True) + base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) + if base_pto != base_object_pto or \ + flags & Py_TPFLAGS_HEAPTYPE: + if not pto.c_tp_new: + pto.c_tp_new = base_pto.c_tp_new + class W_PyCTypeObject(W_TypeObject): def __init__(self, space, pto): - self.pto = pto - bases_w = [] + bases_w = [] # XXX fill dict_w = {} add_operators(space, dict_w, pto) @@ -132,16 +171,28 @@ bases_w or [space.w_object], dict_w) self.__flags__ = _CPYTYPE # mainly disables lookup optimizations -class W_PyCObject(Wrappable): - def __init__(self, space): +class __extend__(W_Root): + __metaclass__ = extendabletype + __slots__ = ("_pyolifeline", ) + _pyolifeline = None + def set_pyolifeline(self, lifeline): + self._pyolifeline = lifeline + def get_pyolifeline(self): + return self._pyolifeline + +class PyOLifeline(object): + def __init__(self, space, pyo): + self.pyo = pyo self.space = space - def set_dual(self, w_obj): - self.w_dual = w_obj - def __del__(self): - Py_DecRef(self.space, self) + if self.pyo: + assert self.pyo.c_ob_refcnt == 0 + _Py_Dealloc(self.space, self.pyo) + self.pyo = lltype.nullptr(PyObject.TO) + # XXX handle borrowed objects here +class GettersAndSetters: def getter(self, space, w_self): return generic_cpy_call( space, self.getset.c_get, w_self, @@ -158,15 +209,44 @@ def member_setter(self, space, w_self, w_value): PyMember_SetOne(space, w_self, self.member, w_value) -class W_PyCObjectDual(W_PyCObject): - def __init__(self, space): - self.space = space +def c_type_descr__call__(space, w_type, __args__): + if isinstance(w_type, W_PyCTypeObject): + pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, pyo) + tp_new = pto.c_tp_new + try: + if tp_new: + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.wrap(key), w_obj) + return generic_cpy_call(space, tp_new, pto, w_args, w_kw) + else: + raise operationerrfmt(space.w_TypeError, + "cannot create '%s' instances", w_type.getname(space, '?')) + finally: + Py_DecRef(space, pyo) + else: + return call__Type(space, w_type, __args__) - def set_pycobject(self, w_pycobject): - self.w_pycobject = ref(w_pycobject) +def c_type_descr__new__(space, w_typetype, w_name, w_bases, w_dict): + # copied from typetype.descr__new__, XXX missing logic: metaclass resolving + w_typetype = _precheck_for_new(space, w_typetype) + + bases_w = space.fixedview(w_bases) + name = space.str_w(w_name) + dict_w = {} + dictkeys_w = space.listview(w_dict) + for w_key in dictkeys_w: + key = space.str_w(w_key) + dict_w[key] = space.getitem(w_dict, w_key) + w_type = space.allocate_instance(W_PyCTypeObject, w_typetype) + W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object], + dict_w) + w_type.ready() + return w_type - def __del__(self): # ok, subclassing isnt so sensible here - pass @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): @@ -202,12 +282,14 @@ def type_dealloc(space, obj): state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) + if not obj_pto.c_tp_name or "C_type" == rffi.charp2str(obj_pto.c_tp_name): + import pdb; pdb.set_trace() type_pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) - Py_DecRef(space, base_pyo) Py_DecRef(space, obj_pto.c_tp_bases) Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, base_pyo) lltype.free(obj_pto.c_tp_name, flavor="raw") obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) @@ -219,7 +301,6 @@ """ Allocates a pto from a w_type which must be a PyPy type. """ state = space.fromcache(State) from pypy.module.cpyext.object import PyObject_dealloc, PyObject_Del - assert not isinstance(w_type, W_PyCTypeObject) assert isinstance(w_type, W_TypeObject) pto = lltype.malloc(PyTypeObject, flavor="raw", zero=True) @@ -264,9 +345,16 @@ PyPyType_Ready(space, pto, w_type) else: pto.c_ob_type = lltype.nullptr(PyObject.TO) + if space.is_w(w_type, space.w_object): + pto.c_tp_basicsize = rffi.sizeof(PyObject.TO) + elif space.is_w(w_type, space.w_type): + pto.c_tp_basicsize = rffi.sizeof(PyTypeObject) + elif space.is_w(w_type, space.w_str): + pto.c_tp_basicsize = rffi.sizeof(PyStringObject.TO) + elif pto.c_tp_base: + pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize - # XXX fill slots in pto - # would look like fixup_slot_dispatchers() + update_all_slots(space, w_type, pto) return pto @cpython_api([PyTypeObjectPtr], rffi.INT_real, error=-1) @@ -283,7 +371,7 @@ base = pto.c_tp_base if not base and not (w_obj is not None and space.is_w(w_obj, space.w_object)): - base_pyo = make_ref(space, space.w_object) + base_pyo = make_ref(space, space.w_object, steal=True) base = pto.c_tp_base = rffi.cast(PyTypeObjectPtr, base_pyo) else: base_pyo = rffi.cast(PyObject, base) @@ -301,7 +389,9 @@ if w_obj is None: PyPyType_Register(space, pto) # missing: - # inherit_special, inherit_slots, setting __doc__ if not defined and tp_doc defined + if base: + inherit_special(space, pto, base) + # inherit_slots, setting __doc__ if not defined and tp_doc defined # inheriting tp_as_* slots # unsupported: # tp_mro, tp_subclasses @@ -324,8 +414,8 @@ w_obj.ready() return 1 -W_PyCObject.typedef = W_ObjectObject.typedef - W_PyCTypeObject.typedef = TypeDef( - 'C_type', W_TypeObject.typedef + 'C_type', W_TypeObject.typedef, + __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), + __new__ = interp2app(c_type_descr__new__), ) From xoraxax at codespeak.net Fri Apr 2 21:36:48 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 21:36:48 +0200 (CEST) Subject: [pypy-svn] r73308 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402193648.0D916282B90@codespeak.net> Author: xoraxax Date: Fri Apr 2 21:36:46 2010 New Revision: 73308 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Suppress certain warnings. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 2 21:36:46 2010 @@ -31,6 +31,9 @@ from pypy.rlib.rstring import rsplit +WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False + + class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset): self.getset = getset @@ -101,7 +104,8 @@ # XXX special case iternext continue if slot_func is None: - os.write(2, method_name + " defined by the type but no slot function defined!\n") + if WARN_ABOUT_MISSING_SLOT_FUNCTIONS: + os.write(2, method_name + " defined by the type but no slot function defined!\n") continue if method_name == "__new__" and "bar" in repr(w_obj): import pdb; pdb.set_trace() From xoraxax at codespeak.net Fri Apr 2 22:14:30 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 22:14:30 +0200 (CEST) Subject: [pypy-svn] r73309 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100402201430.D6167282B90@codespeak.net> Author: xoraxax Date: Fri Apr 2 22:14:29 2010 New Revision: 73309 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix skipped test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 2 22:14:29 2010 @@ -82,6 +82,7 @@ mod = compile_module(name, **kwds) api.load_extension_module(self.space, mod, name) + self.name = name return self.space.getitem( self.space.sys.get('modules'), self.space.wrap(name)) @@ -120,15 +121,17 @@ def teardown_method(self, func): try: w_mod = self.space.getitem(self.space.sys.get('modules'), - self.space.wrap('foo')) + self.space.wrap(self.name)) self.space.delitem(self.space.sys.get('modules'), - self.space.wrap('foo')) + self.space.wrap(self.name)) Py_DecRef(self.space, w_mod) state = self.space.fromcache(State) for w_obj in state.non_heaptypes: Py_DecRef(self.space, w_obj) except OperationError: pass + except AttributeError: + pass state = self.space.fromcache(State) if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -267,7 +270,6 @@ A module can use the PyMODINIT_FUNC macro to declare or define its module initializer function. """ - skip("This leaks references for some reason") module = self.import_module(name='modinit') assert module.__name__ == 'modinit' From trundle at codespeak.net Fri Apr 2 22:21:17 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Fri, 2 Apr 2010 22:21:17 +0200 (CEST) Subject: [pypy-svn] r73310 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100402202117.1FA99282B90@codespeak.net> Author: trundle Date: Fri Apr 2 22:21:15 2010 New Revision: 73310 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Add T_OBJECT and T_OBJECT_EX members and handle deletion of members. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Fri Apr 2 22:21:15 2010 @@ -22,6 +22,10 @@ /* Types */ #define T_INT 1 +#define T_OBJECT 6 +#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError + when the value is NULL, instead of + converting to None. */ /* Flags */ #define READONLY 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Fri Apr 2 22:21:15 2010 @@ -2,20 +2,31 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.api import ADDR, cpython_api +from pypy.module.cpyext.api import ADDR, PyObjectP, cpython_api from pypy.module.cpyext.intobject import PyInt_AsLong from pypy.module.cpyext.pyerrors import PyErr_Occurred -from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, from_ref, make_ref from pypy.module.cpyext.typeobjectdefs import PyMemberDef @cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject) def PyMember_GetOne(space, obj, w_member): - ptr = rffi.cast(ADDR, obj) + addr = rffi.cast(ADDR, obj) + addr += w_member.c_offset member_type = rffi.cast(lltype.Signed, w_member.c_type) if member_type == structmemberdefs.T_INT: - result = rffi.cast(rffi.INTP, ptr + w_member.c_offset) + result = rffi.cast(rffi.INTP, addr) w_result = space.wrap(result[0]) + elif member_type in [structmemberdefs.T_OBJECT, + structmemberdefs.T_OBJECT_EX]: + obj_ptr = rffi.cast(PyObjectP, addr) + if obj_ptr[0]: + w_result = from_ref(space, obj_ptr[0]) + else: + if member_type == structmemberdefs.T_OBJECT_EX: + w_name = space.wrap(rffi.charp2str(w_member.c_name)) + raise OperationError(space.w_AttributeError, w_name) + w_result = space.w_None else: raise OperationError(space.w_SystemError, space.wrap("bad memberdescr type")) @@ -24,12 +35,33 @@ @cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, error=-1) def PyMember_SetOne(space, obj, w_member, w_value): - ptr = rffi.cast(ADDR, obj) + addr = rffi.cast(ADDR, obj) + addr += w_member.c_offset member_type = rffi.cast(lltype.Signed, w_member.c_type) + flags = rffi.cast(lltype.Signed, w_member.c_flags) + + if flags & structmemberdefs.READONLY: + raise OperationError(space.w_TypeError, + space.wrap("readonly attribute")) + elif w_value is None: + if member_type == structmemberdefs.T_OBJECT_EX: + if not rffi.cast(PyObjectP, addr)[0]: + w_name = space.wrap(rffi.charp2str(w_member.c_name)) + raise OperationError(space.w_AttributeError, w_name) + elif member_type != structmemberdefs.T_OBJECT: + raise OperationError(space.w_TypeError, + space.wrap("can't delete numeric/char attribute")) + if member_type == structmemberdefs.T_INT: w_long_value = PyInt_AsLong(space, w_value) - array = rffi.cast(rffi.INTP, ptr + w_member.c_offset) + array = rffi.cast(rffi.INTP, addr) array[0] = rffi.cast(rffi.INT, w_long_value) + elif member_type in [structmemberdefs.T_OBJECT, + structmemberdefs.T_OBJECT_EX]: + array = rffi.cast(PyObjectP, addr) + if array[0]: + Py_DecRef(space, array[0]) + array[0] = make_ref(space, w_value) else: raise OperationError(space.w_SystemError, space.wrap("bad memberdescr type")) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Fri Apr 2 22:21:15 2010 @@ -1,3 +1,5 @@ T_INT = 1 +T_OBJECT = 6 +T_OBJECT_EX = 16 READONLY = RO = 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Fri Apr 2 22:21:15 2010 @@ -4,6 +4,7 @@ typedef struct { PyObject_HEAD int foo; /* the context holder */ + PyObject *foo_object; } fooobject; static PyTypeObject footype; @@ -18,6 +19,7 @@ return NULL; foop->foo = 42; + foop->foo_object = NULL; return foop; } @@ -99,6 +101,10 @@ {"int_member_readonly", T_INT, offsetof(fooobject, foo), READONLY, "A helpful docstring."}, {"broken_member", 0xaffe, 0, 0, ""}, + {"object_member", T_OBJECT, offsetof(fooobject, foo_object), 0, + "A Python object."}, + {"object_member_ex", T_OBJECT_EX, offsetof(fooobject, foo_object), 0, + "A Python object."}, {NULL} /* Sentinel */ }; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Fri Apr 2 22:21:15 2010 @@ -33,9 +33,27 @@ assert obj.int_member == 23 obj.int_member = 42 raises(TypeError, "obj.int_member = 'not a number'") + raises(TypeError, "del obj.int_member") raises(TypeError, "obj.int_member_readonly = 42") + exc = raises(TypeError, "del obj.int_member_readonly") + assert "readonly" in str(exc.value) raises(SystemError, "obj.broken_member") raises(SystemError, "obj.broken_member = 42") + + assert obj.object_member is None + obj.object_member = "hello" + assert obj.object_member == "hello" + del obj.object_member + del obj.object_member + assert obj.object_member is None + raises(AttributeError, "obj.object_member_ex") + obj.object_member_ex = None + assert obj.object_member_ex is None + obj.object_member_ex = 42 + assert obj.object_member_ex == 42 + del obj.object_member_ex + raises(AttributeError, "del obj.object_member_ex") + a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message skip("In Progress") Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 2 22:21:15 2010 @@ -61,9 +61,10 @@ if doc: doc = rffi.charp2str(getset.c_doc) get = GettersAndSetters.member_getter.im_func + del_ = GettersAndSetters.member_delete.im_func if not (flags & structmemberdefs.READONLY): set = GettersAndSetters.member_setter.im_func - GetSetProperty.__init__(self, get, set, None, doc, + GetSetProperty.__init__(self, get, set, del_, doc, cls=None, use_closure=True, # XXX cls? tag="cpyext_2") @@ -210,6 +211,9 @@ def member_getter(self, space, w_self): return PyMember_GetOne(space, w_self, self.member) + def member_delete(self, space, w_self): + PyMember_SetOne(space, w_self, self.member, None) + def member_setter(self, space, w_self, w_value): PyMember_SetOne(space, w_self, self.member, w_value) From xoraxax at codespeak.net Fri Apr 2 22:36:22 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 2 Apr 2010 22:36:22 +0200 (CEST) Subject: [pypy-svn] r73311 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402203622.41B36282B90@codespeak.net> Author: xoraxax Date: Fri Apr 2 22:36:20 2010 New Revision: 73311 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Some translation fixes. Currently translation is blocked by getargs code. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 2 22:36:20 2010 @@ -57,7 +57,7 @@ pto = rffi.cast(PyTypeObjectPtr, w_type_pyo) # Don't increase refcount for non-heaptypes if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, pto) + Py_DecRef(space, w_type_pyo) basicsize = pto.c_tp_basicsize py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, flavor="raw", zero=True) @@ -132,6 +132,7 @@ def Py_DecRef(space, obj): if not obj: return + assert lltype.typeOf(obj) == PyObject from pypy.module.cpyext.typeobject import string_dealloc, W_PyCTypeObject obj.c_ob_refcnt -= 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 2 22:36:20 2010 @@ -29,6 +29,7 @@ from pypy.module.cpyext.slotdefs import slotdefs from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.rstring import rsplit +from pypy.rlib.objectmodel import we_are_translated WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -108,7 +109,7 @@ if WARN_ABOUT_MISSING_SLOT_FUNCTIONS: os.write(2, method_name + " defined by the type but no slot function defined!\n") continue - if method_name == "__new__" and "bar" in repr(w_obj): + if not we_are_translated() and method_name == "__new__" and "bar" in repr(w_obj): import pdb; pdb.set_trace() slot_func_helper = llhelper(slot_func.api_func.functype, slot_func.api_func.get_wrapper(space)) @@ -236,6 +237,7 @@ finally: Py_DecRef(space, pyo) else: + w_type = _precheck_for_new(space, w_type) return call__Type(space, w_type, __args__) def c_type_descr__new__(space, w_typetype, w_name, w_bases, w_dict): @@ -290,8 +292,6 @@ def type_dealloc(space, obj): state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) - if not obj_pto.c_tp_name or "C_type" == rffi.charp2str(obj_pto.c_tp_name): - import pdb; pdb.set_trace() type_pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) From exarkun at codespeak.net Sat Apr 3 00:34:30 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 3 Apr 2010 00:34:30 +0200 (CEST) Subject: [pypy-svn] r73312 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100402223430.CB14D282B90@codespeak.net> Author: exarkun Date: Sat Apr 3 00:34:28 2010 New Revision: 73312 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: Skip PyErr_SetFromErrno until it can be written in a valid way Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Sat Apr 3 00:34:28 2010 @@ -70,6 +70,9 @@ def test_SetFromErrno(self): + skip("The test does not set the errno in a way which " + "untranslated pypy can actually notice") + import errno module = self.import_extension('foo', [ From fijal at codespeak.net Sat Apr 3 00:42:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 00:42:24 +0200 (CEST) Subject: [pypy-svn] r73313 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402224224.25C45282B90@codespeak.net> Author: fijal Date: Sat Apr 3 00:42:22 2010 New Revision: 73313 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: I think we want signed here Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 3 00:42:22 2010 @@ -264,7 +264,7 @@ (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) -VA_TP_LIST = {'int': rffi.INT, +VA_TP_LIST = {'int': lltype.Signed, 'PyObject*': PyObject, 'int*': rffi.INTP} From fijal at codespeak.net Sat Apr 3 01:23:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:23:54 +0200 (CEST) Subject: [pypy-svn] r73314 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100402232354.A054C282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:23:52 2010 New Revision: 73314 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Write down a test for PyString_FromFormatV, fails for irrelevant reasons (I think) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c Sat Apr 3 01:23:52 2010 @@ -2,7 +2,7 @@ #include "Python.h" PyObject * -PyString_FromFormatV(const char *format, va_list vargs) +PyString_FromFormatV(char *format, va_list vargs) { va_list count; Py_ssize_t n = 0; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Sat Apr 3 01:23:52 2010 @@ -15,8 +15,7 @@ Py_ssize_t size; } PyStringObject; -PyObject * -PyString_FromFormatV(const char *format, va_list vargs); +PyObject *PyString_FromFormatV(char *format, va_list vargs); #ifdef __cplusplus } Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:23:52 2010 @@ -51,7 +51,7 @@ return space.int_w(space.len(w_obj)) @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyString_Resize(space, string, newsize): +def _PyString_Resize(space, w_string, newsize): """A way to resize a string object even though it is "immutable". Only use this to build up a brand new string object; don't use this if the string may already be known in other parts of the code. It is an error to call this function if the @@ -64,4 +64,5 @@ This function used an int type for newsize. This might require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError + import pdb + pdb.set_trace() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Sat Apr 3 01:23:52 2010 @@ -87,7 +87,7 @@ self.space.sys.get('modules'), self.space.wrap(name)) - def import_extension(self, modname, functions): + def import_extension(self, modname, functions, prologue=""): methods_table = [] codes = [] @@ -103,7 +103,7 @@ """ % (cfuncname, code) codes.append(func_code) - body = "\n".join(codes) + """ + body = prologue + "\n".join(codes) + """ static PyMethodDef methods[] = { %s { NULL } Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Sat Apr 3 01:23:52 2010 @@ -96,15 +96,34 @@ s = module.getstring() assert s == 'test' + def test_py_string_as_string(self): + module = self.import_extension('foo', [ + ("string_as_string", "METH_VARARGS", + ''' + return PyString_FromStringAndSize(PyString_AsString( + PyTuple_GetItem(args, 0)), 4); + ''' + )]) + assert module.string_as_string("huheduwe") == "huhe" + def test_format_v(self): - skip("unsupported yet, think how to fak va_list") module = self.import_extension('foo', [ ("test_string_format_v", "METH_VARARGS", ''' - return PyString_FromFormatV("bla %d ble %s", + return helper("bla %d ble %s\\n", PyInt_AsLong(PyTuple_GetItem(args, 0)), PyString_AsString(PyTuple_GetItem(args, 1))); ''' ) - ]) - pass + ], prologue=''' + PyObject* helper(char* fmt, ...) + { + va_list va; + va_start(va, fmt); + PyObject* res = PyString_FromFormatV(fmt, va); + va_end(va); + return res; + } + ''') + res = module.test_string_format_v(1, "xyz") + print res From magcius at codespeak.net Sat Apr 3 01:46:33 2010 From: magcius at codespeak.net (magcius at codespeak.net) Date: Sat, 3 Apr 2010 01:46:33 +0200 (CEST) Subject: [pypy-svn] r73315 - in pypy/branch/avm/pypy/translator: . avm1 avm1/.ropeproject avm1/test avm2 avm2/test tool Message-ID: <20100402234633.B1291282B90@codespeak.net> Author: magcius Date: Sat Apr 3 01:46:31 2010 New Revision: 73315 Added: pypy/branch/avm/pypy/translator/avm2/library.py Removed: pypy/branch/avm/pypy/translator/avm1/.ropeproject/ pypy/branch/avm/pypy/translator/avm1/records_flymake.py pypy/branch/avm/pypy/translator/avm1/test/harness_flymake.py Modified: pypy/branch/avm/pypy/translator/avm1/avm1.py pypy/branch/avm/pypy/translator/avm1/avm1gen.py pypy/branch/avm/pypy/translator/avm1/types_.py pypy/branch/avm/pypy/translator/avm2/avm2gen.py pypy/branch/avm/pypy/translator/avm2/class_.py pypy/branch/avm/pypy/translator/avm2/conftest.py pypy/branch/avm/pypy/translator/avm2/function.py pypy/branch/avm/pypy/translator/avm2/genavm.py pypy/branch/avm/pypy/translator/avm2/metavm.py pypy/branch/avm/pypy/translator/avm2/runtime.py pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py pypy/branch/avm/pypy/translator/avm2/test/runtest.py pypy/branch/avm/pypy/translator/avm2/types_.py pypy/branch/avm/pypy/translator/driver.py pypy/branch/avm/pypy/translator/interactive.py pypy/branch/avm/pypy/translator/tool/autopath.py Log: More work on PyPy-AVM Modified: pypy/branch/avm/pypy/translator/avm1/avm1.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm1/avm1.py (original) +++ pypy/branch/avm/pypy/translator/avm1/avm1.py Sat Apr 3 01:46:31 2010 @@ -1,4 +1,3 @@ - # AVM1 = ActionScript Virtual Machine 1 # Used for ActionScript 1 and 2 @@ -27,28 +26,28 @@ _global="preload_global") class Action(object): - + ACTION_NAME = "NotImplemented" ACTION_ID = 0x00 offset = 0 label_name = "" - + def serialize(self): inner_data = self.gen_data() outer_data = self.gen_outer_data() header = struct.pack(" 0 and action.ACTION_NAME == "ActionPush" and self.actions[-1].ACTION_NAME == "ActionPush": old_action = self.actions[-1] @@ -171,13 +170,13 @@ self.actions.pop() self.current_offset -= 1 # len(ShortAction) is 1 return None - + if not isinstance(action, Block): # Don't add block length until we've finalized. self.current_offset += len(action) - + self.actions.append(action) return action - + def serialize(self): if not self._sealed: raise SealedBlockError("Block must be sealed before it can be serialized") @@ -201,7 +200,7 @@ name = Block.AUTO_LABEL_TEMPLATE % self.label_count self.labels[name] = -1 return name - + def set_label_here(self, name): self.labels[name] = self.current_offset @@ -245,7 +244,7 @@ self.function_name = name self.params = parameters self.preload_register_count = 1 # Start at 1. - + # Flags self.registers = [None] self.preload_parent = False @@ -261,31 +260,31 @@ for name in parameters: self.registers.append(name) - + def eval_flags(self): - + # According to the docs, this is the order of register allocation. if self.preload_this and "this" not in self.registers: self.suppress_this = False self.registers.insert(1, "this") - + if self.preload_args and "arguments" not in self.registers: self.suppress_args = False self.registers.insert(2, "arguments") - + if self.preload_super and "super" not in self.registers: self.suppress_super = False self.registers.insert(3, "super") - + if self.preload_root and "_root" not in self.registers: self.registers.insert(4, "_root") - + if self.preload_parent and "_parent" not in self.registers: self.registers.insert(5, "_parent") - + if self.preload_global and "_global" not in self.registers: self.registers.insert(6, "_global") - + def gen_data(self): bits = BitStream() @@ -299,15 +298,15 @@ bits.write_bit(self.preload_this) bits.zero_fill(7) # skip over 7 Reserved bits bits.write_bit(self.preload_global) - + self.block_data = Block.serialize(self) bytes = [self.function_name, "\0", struct.pack("HB", len(self.params), len(self.registers)), bits.serialize()] - + for name in self.params: bytes += [chr(self.registers.index(name)), name, "\0"] - + bytes += [struct.pack("H", len(self.block_data))] return "".join(bytes) @@ -417,11 +416,11 @@ ACTION_ID = 0x96 USE_CONSTANTS = False - + def __init__(self, *args): self.values = [] self.add_element(*args) - + def add_element(self, element): if hasattr(element, "__iter__") and not isinstance(element, (basestring, tuple)): for t in element: @@ -431,14 +430,14 @@ element = (None, element) assert isinstance(element, tuple) self.values.append(element) - + def get_block_props_early(self, block): if not ActionPush.USE_CONSTANTS: return for index, (value, type) in enumerate(self.values): if type == STRING: constant_index = block.constants.add_constant(value) self.values[index] = (constant_index, CONSTANT8 if constant_index < 256 else CONSTANT16) - + def gen_data(self): bytes = [] for value, type in self.values: @@ -476,7 +475,7 @@ def __init__(self, catch_object, try_block=None, catch_block=None, finally_block=None): self.catch_object = catch_object - + self.try_block = try_block or Block() self.catch_block = catch_block or Block() self.finally_block = finally_block or Block() @@ -513,7 +512,7 @@ def gen_data(self): return struct.pack("HB", self.index, self.skip_count) - + class ActionWaitForFrame2(Action): ACTION_NAME = "ActionWaitForFrame2" ACTION_ID = 0x8d @@ -527,10 +526,10 @@ class ActionWith(Action): ACTION_NAME = "ActionWith" ACTION_ID = 0x94 - + def __init__(self, with_block): self.block = with_block or Block() - + def gen_data(self): return struct.pack("H", len(self.block)) + self.block.serialize() @@ -541,13 +540,12 @@ return ''.join('_' + c.lower() if c.isupper() else c for c in name)[1:] def make_short_action(value, name, push_count=0): - def __len__(self): return 1 # 1 (Action ID) - + def serialize(self): return chr(self.ACTION_ID) - + act = type(name, (Action,), dict(ACTION_ID=value, ACTION_NAME=name, push_count=push_count, __len__=__len__, serialize=serialize)) Modified: pypy/branch/avm/pypy/translator/avm1/avm1gen.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm1/avm1gen.py (original) +++ pypy/branch/avm/pypy/translator/avm1/avm1gen.py Sat Apr 3 01:46:31 2010 @@ -50,8 +50,17 @@ super(PyPyAVM1Gen, self).load(v, *args) - def new(self, TYPE): if isinstance(TYPE, ootype.List): self.oonewarray(None) - + + def push_primitive_constant(self, TYPE, value): + if TYPE is ootype.Void: + self.push_null() + elif TYPE is ootype.String: + if value._str is None: + self.push_null() + else: + self.push_const(value._str) + else: + self.push_const(value) Modified: pypy/branch/avm/pypy/translator/avm1/types_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm1/types_.py (original) +++ pypy/branch/avm/pypy/translator/avm1/types_.py Sat Apr 3 01:46:31 2010 @@ -1,20 +1,17 @@ -from pypy.translator.avm import avm1 from pypy.rlib.rarithmetic import r_longlong, r_ulonglong from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem import lltype +from mech.fusion.avm1 import types_, actions + _pytype_to_avm1 = { - str: avm1.STRING, - unicode: avm1.STRING, - int: avm1.INTEGER, - long: avm1.INTEGER, - r_longlong: avm1.INTEGER, - r_ulonglong: avm1.INTEGER, - bool: avm1.BOOLEAN, - float: avm1.DOUBLE, + r_longlong: types_.INTEGER, + r_ulonglong: types_.INTEGER, } +_pytype_to_avm1.update(types_._pytype_to_avm1) + def pytype_to_avm1(value): return (value, _pytype_to_avm1[type(value)]) Modified: pypy/branch/avm/pypy/translator/avm2/avm2gen.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/avm2gen.py (original) +++ pypy/branch/avm/pypy/translator/avm2/avm2gen.py Sat Apr 3 01:46:31 2010 @@ -1,4 +1,3 @@ - """ backend generator routines """ @@ -14,32 +13,19 @@ from pypy.translator.oosupport.constant import push_constant from pypy.translator.oosupport.function import render_sub_op - from itertools import chain class PyPyAvm2ilasm(avm2gen.Avm2ilasm, Generator): - - def __init__(self, db, abc): - super(PyPyAvm2ilasm, self).__init__(abc) + def __init__(self, db, abc, optimize=False): + super(PyPyAvm2ilasm, self).__init__(abc, optimize=optimize) self.db = db self.cts = db.genoo.TypeSystem(db) def _get_type(self, TYPE): - return self.cts.lltype_to_cts(TYPE) - - def _get_class_context(self, name, DICT): - class_desc = query.get_class_desc(name) - if class_desc: - BaseType = class_desc.BaseType - if '.' in BaseType: - ns, name = class_desc.BaseType.rsplit('.', 1) - else: - ns, name = '', BaseType - class_desc.super_name = constants.packagedQName(ns, name) - class_desc.name = constants.packagedQName(class_desc.Package, class_desc.ShortName) - return class_desc - else: - return super(PyPyAvm2ilasm, self).get_class_context(name, DICT) + t = self.cts.lltype_to_cts(TYPE) + if t: + return t.multiname() + return super(PyPyAvm2ilasm, self)._get_type(TYPE) def load(self, v, *args): if isinstance(v, flowmodel.Variable): @@ -57,29 +43,21 @@ for e in args: self.load(e) - # def prepare_call_oostring(self, OOTYPE): - # self.I(instructions.findpropstrict(types._str_qname)) - - # def call_oostring(self, OOTYPE): - # self.I(instructions.callproperty(types._str_qname, 1)) - - # call_oounicode = call_oostring - # prepare_call_oounicode = prepare_call_oostring - def oonewarray(self, TYPE, length=1): self.load(types.vec_qname) self.load(self.cts.lltype_to_cts(TYPE.ITEM)) self.I(instructions.applytype(1)) self.load(length) self.I(instructions.construct(1)) - self.I(instructions.coerce(self.cts.lltype_to_cts(TYPE).multiname())) - + self.I(instructions.coerce(self.cts.lltype_to_cts(TYPE))) + def push_primitive_constant(self, TYPE, value): if TYPE is ootype.Void: self.push_null() elif TYPE is ootype.String: if value._str is None: self.push_null() + self.downcast(types.types.string) else: self.push_const(value._str) else: @@ -87,7 +65,7 @@ def new(self, TYPE): # XXX: assume no args for now - t = self.cts.lltype_to_cts(TYPE).multiname() + t = self._get_type(TYPE) self.emit('findpropstrict', t) self.emit('constructprop', t, 0) @@ -102,23 +80,20 @@ self.I(instructions.getproperty(constants.MultinameL( constants.PROP_NAMESPACE_SET))) - def array_length(self): - self.I(instructions.getproperty(constants.QName("length"))) - - def call_graph(self, graph, func_name=None): + def call_graph(self, graph, args=[]): """ Call a graph. """ - if func_name is None: - self.db.pending_function(graph) - func_name = func_name or graph.name + self.db.pending_function(graph) namespace = getattr(graph.func, '_namespace_', None) + numargs = len(args) if namespace: - qname = constants.packagedQName(namespace, func_name) + qname = constants.packagedQName(namespace, graph.name) else: - qname = constants.QName(func_name) + qname = constants.QName(graph.name) self.emit('findpropstrict', qname) - self.emit('callproperty', qname) + self.load(*args) + self.emit('callproperty', qname, numargs) def store(self, v): """ @@ -133,9 +108,3 @@ self.push_var(v.name) push_arg = push_local - - def new(self, TYPE): - # XXX: assume no args for now - TYPE = self._get_type(TYPE) - self.emit('findpropstrict', TYPE) - self.emit('constructprop', TYPE, 0) Modified: pypy/branch/avm/pypy/translator/avm2/class_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/class_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/class_.py Sat Apr 3 01:46:31 2010 @@ -2,7 +2,7 @@ from pypy.translator.cli.node import Node from pypy.translator.oosupport.constant import push_constant -from mech.fusion.avm2 import constants as c, traits +from mech.fusion.avm2 import constants, traits from pypy.translator.avm2 import types_ as types try: @@ -45,12 +45,12 @@ def get_base_class(self): base_class = self.INSTANCE._superclass if self.INSTANCE is self.db.genoo.EXCEPTION: - return c.QName("Error") + return constants.QName("Error") if self.is_root(base_class): - return c.QName("Object") + return constants.QName("Object") else: ns, name = self.db.class_name(base_class).rsplit('::', 1) - return c.packagedQName(ns, name) + return constants.packagedQName(ns, name) def is_abstract(self): return False # XXX @@ -80,14 +80,13 @@ return self.ilasm = ilasm - self.gen = CLIBaseGenerator(self.db, ilasm) - ilasm.begin_class(c.packagedQName(self.namespace, self.name), self.get_base_class()) + ilasm.begin_class(constants.packagedQName(self.namespace, self.name), self.get_base_class()) for f_name, (f_type, f_default) in self.INSTANCE._fields.iteritems(): cts_type = self.cts.lltype_to_cts(f_type) f_name = self.cts.escape_name(f_name) if cts_type != types.types.void: - ilasm.context.add_instance_trait(traits.AbcSlotTrait(c.QName(f_name), cts_type.multiname())) + ilasm.context.add_instance_trait(traits.AbcSlotTrait(constants.QName(f_name), cts_type.multiname())) self._ctor() self._toString() @@ -108,7 +107,6 @@ while context: if m_name in context._methods: f.override = True - print "Overriding", m_name break context = context._superclass @@ -121,16 +119,16 @@ if ARG is not ootype.Void] returntype = self.cts.lltype_to_cts(METH.RESULT) ilasm.begin_method(m_name, arglist, returntype) - ilasm.emit('findpropstrict', c.QName("Error")) + ilasm.emit('findpropstrict', constants.QName("Error")) ilasm.push_const("Abstract method %s::%s called" % (self.name, m_name)) - ilasm.emit('constructprop', c.QName("Error"), 1) - ilasm.emit('throw') + ilasm.emit('constructprop', constants.QName("Error"), 1) + ilasm.throw() ilasm.exit_context() ilasm.exit_context() def _ctor(self): - self.ilasm.context.make_iinit() + self.ilasm.begin_constructor() # set default values for fields default_values = self.INSTANCE._fields.copy() default_values.update(self.INSTANCE._overridden_defaults) @@ -144,9 +142,8 @@ self.ilasm.push_this() push_constant(self.db, F_TYPE, f_default, self.gen) # class_name = self.db.class_name(INSTANCE_DEF) - self.ilasm.emit('setproperty', c.QName(f_name)) - - self.ilasm.exit_context() + self.ilasm.set_field(f_name) + self.ilasm.end_constructor() def _toString(self): if self.is_root(self.INSTANCE._superclass): @@ -157,6 +154,5 @@ wrapper = "Exception" if self.exception else "Instance" self.ilasm.begin_method('toString', [], types.types.string, override=override) self.ilasm.load("%sWrapper('%s')" % (wrapper, self.name)) - self.ilasm.emit('returnvalue') - self.ilasm.exit_context() - + self.ilasm.return_value() + self.ilasm.end_method() Modified: pypy/branch/avm/pypy/translator/avm2/conftest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/conftest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/conftest.py Sat Apr 3 01:46:31 2010 @@ -1,6 +1,8 @@ def pytest_addoption(parser): group = parser.getgroup("pypy-tamarin options") group.addoption('--swf', action="store_const", const="swf", dest="tamtarget", default="swf", - help="generate a swf and abc and use a browsertest to run") + help="generate a .swf and .abc and use a browsertest and Flash Player to run") group.addoption('--tamarin', action="store_const", const="tamarin", dest="tamtarget", - help="generate an abc that uses tamarin") + help="generate an abc that uses Tamarin") + group.addoption('--no-mf-optimize', action="store_false", default=True, dest="mf_optim", + help="don't do simple MF optimizations") Modified: pypy/branch/avm/pypy/translator/avm2/function.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/function.py (original) +++ pypy/branch/avm/pypy/translator/avm2/function.py Sat Apr 3 01:46:31 2010 @@ -1,3 +1,4 @@ + from functools import partial from pypy.objspace.flow import model as flowmodel @@ -5,19 +6,18 @@ from pypy.rpython.lltypesystem.lltype import Void from pypy.translator.oosupport.function import Function as OOFunction from pypy.translator.cli.node import Node -from pypy.translator.avm2 import types_ as types from mech.fusion.avm2 import constants class Function(OOFunction, Node): - + auto_propagate_exceptions = True - def __init__(self, *args, **kwargs): - OOFunction.__init__(self, *args, **kwargs) - + def __init__(self, db, graph, name=None, is_method=False, is_entrypoint=False): + OOFunction.__init__(self, db, graph, name, is_method, is_entrypoint) + if hasattr(self.db.genoo, 'exceptiontransformer'): self.auto_propagate_exceptions = False - + namespace = getattr(self.graph.func, '_namespace_', None) if namespace: if '.' in namespace: @@ -28,13 +28,13 @@ else: self.namespace = None self.classname = None - + self.override = False - + def _create_generator(self, ilasm): ilasm.db = self.db return ilasm - + def record_ll_meta_exc(self, ll_meta_exc): # record the type only if it doesn't belong to a native_class ll_exc = ll_meta_exc._INSTANCE @@ -50,21 +50,21 @@ if self.is_method: self.args = self.args[1:] - + returntype, returnvar = self.cts.llvar_to_cts(self.graph.getreturnvar()) if self.classname: self.generator.begin_class(constants.packagedQName(self.namespace, self.classname)) - + self.generator.begin_method(self.name, self.args, returntype, static=not self.is_method, override=self.override) - + + self.declare_locals() + def end_render(self): - # if self.generator.scope.islabel: - # self.generator.exit_scope() if self.classname: self.generator.exit_context() self.generator.exit_context() - + def render_return_block(self, block): return_var = block.inputargs[0] if return_var.concretetype is Void: @@ -76,6 +76,11 @@ def set_label(self, label): return self.generator.set_label(label) + def declare_locals(self): + for TYPE, name in set(self.locals): + TYPE.load_default(self.generator) + self.generator.store_var(name) + def _trace_enabled(self): return True @@ -88,38 +93,39 @@ def _render_op(self, op): print "Rendering op:", op super(Function, self)._render_op(op) - - def _setup_link(self, link): - target = link.target - linkvars = [] - for to_load, to_store in zip(link.args, target.inputargs): - if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: - continue - if to_load.concretetype is ootype.Void: - continue - linkvars.append((to_load, to_store)) - - # after SSI_to_SSA it can happen to have to_load = [a, b] and - # to_store = [b, c]. If we store each variable sequentially, - # 'b' would be overwritten before being read. To solve, we - # first load all the values on the stack, then store in the - # appropriate places. - - if self._trace_enabled(): - self._trace('link', writeline=True) - for to_load, to_store in linkvars: - self._trace_value('%s <-- %s' % (to_store, to_load), to_load) - self._trace('', writeline=True) - - for to_load, to_store in linkvars: - self.generator.load(to_load) - self.generator.store(to_store) - + ## def _setup_link(self, link): + ## target = link.target + ## linkvars = [] + ## for to_load, to_store in zip(link.args, target.inputargs): + ## if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: + ## continue + ## if to_load.concretetype is ootype.Void: + ## continue + ## linkvars.append((to_load, to_store)) + + ## # after SSI_to_SSA it can happen to have to_load = [a, b] and + ## # to_store = [b, c]. If we store each variable sequentially, + ## # 'b' would be overwritten before being read. To solve, we + ## # first load all the values on the stack, then store in the + ## # appropriate places. + + ## if self._trace_enabled(): + ## self._trace('link', writeline=True) + ## for to_load, to_store in linkvars: + ## self._trace_value('%s <-- %s' % (to_store, to_load), to_load) + ## self._trace('', writeline=True) + + ## for to_load, to_store in linkvars: + ## self.generator.load(to_load) + + ## for to_load, to_store in reversed(linkvars): + ## self.generator.store(to_store) + def begin_try(self, cond): if cond: self.ilasm.begin_try() - + def end_try(self, target_label, cond): if cond: self.ilasm.end_try() Modified: pypy/branch/avm/pypy/translator/avm2/genavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/genavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/genavm.py Sat Apr 3 01:46:31 2010 @@ -1,4 +1,3 @@ - import py from mech.fusion.avm2.abc_ import AbcFile @@ -11,7 +10,7 @@ from pypy.translator.avm2.types_ import Avm2TypeSystem class GenAVM2(GenOO): - + opcodes = opcodes Function = TamarinFunction Database = LowLevelDatabase @@ -24,7 +23,7 @@ RecordConst = Avm2RecordConst ListConst = Avm2ArrayListConst ArrayConst = Avm2ArrayListConst - + def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) self.const_stat = str(tmpdir.join('const_stat')) @@ -35,10 +34,10 @@ clsdef = bk.getuniqueclassdef(Exception) ll_Exception = rtyper.exceptiondata.get_standard_ll_exc_instance(rtyper, clsdef) self.EXCEPTION = ll_Exception._inst._TYPE - + def create_assembler(self): self.abc = AbcFile() - return PyPyAvm2ilasm(self.db, self.abc) + return PyPyAvm2ilasm(self.db, self.abc, True) def generate_source(self): if self.ilasm is None: @@ -50,7 +49,3 @@ def serialize_abc(self): return self.abc.serialize() - - # Don't do treebuilding stuff - # def stack_optimization(self): - # pass Added: pypy/branch/avm/pypy/translator/avm2/library.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/library.py Sat Apr 3 01:46:31 2010 @@ -0,0 +1,80 @@ + +from pypy.rpython.ootypesystem import ootype + +from mech.fusion.avm2.constants import QName, packagedQName, TYPE_MULTINAME_TypeName +from mech.fusion.avm2.query import ClassDesc +from mech.fusion.avm2.library import Library +from mech.fusion.avm2.playerglobal.flash.utils import Vector + +from pypy.translator.avm2.types_ import vec_qname + +## Monkey Patching! + +ClassDesc._nativeclass = None + +class PyPyLibrary(Library): + def resolve_class(self, TYPE): + if self.has_type(TYPE): + return self.get_type(TYPE) + if playerglobal_lib.has_type(TYPE): + return self.get_type(TYPE) + if TYPE.KIND == TYPE_MULTINAME_TypeName and TYPE.name == vec_qname: + assert len(TYPE.types) == 1 + return Vector[TYPE.types[0]] + if getattr(TYPE, "multiname", None): + return TYPE.multiname() + + def convert_classdesc(self, classdesc): + resolve = self.resolve_class + from pypy.translator.avm2.runtime import NativeClass, NativeInstance + from pypy.translator.avm2.runtime import _overloaded_static_meth, _static_meth + + if classdesc._nativeclass is not None: + return classdesc._nativeclass + + TYPE = NativeInstance(classdesc.Package, classdesc.ShortName, None, {}, {}) + Class = NativeClass(TYPE, {}, {}) + classdesc._nativeclass = Class + if classdesc.FullName == QName('Object'): + TYPE._set_superclass(ootype.ROOT) + else: + BASETYPE = resolve(classdesc.BaseType) + TYPE._set_superclass(BASETYPE) + + TYPE._isArray = classdesc.IsArray + if classdesc.IsArray: + TYPE._ELEMENT = resolve(classdesc.ElementType) + + # add both static and instance methods, and static fields + static_meths = self.group_methods(classdesc.StaticMethods, + _overloaded_static_meth, _static_meth, ootype.StaticMethod) + meths = self.group_methods(classdesc.Methods, ootype.overload, + ootype.meth, ootype.Meth) + Class._add_methods(static_meths) + Class._add_static_fields(dict((name, + resolve(t)) for name, t in classdesc.StaticFields])) + Class._add_static_fields(dict((name, + resolve(t)) for name, t, g, s in classdesc.StaticProperties)) + TYPE._add_methods(meths) + TYPE._add_fields(dict((name, resolve(t)) for name, t in classdesc.Fields)) + TYPE._add_fields(dict((name, resolve(t)) for name, t, g, s in classdesc.Properties)) + return Class + + def group_methods(self, methods, overload, meth, Meth): + from pypy.translator.avm2.runtime import OverloadingResolver + groups = {} + for name, args, result, AS3 in methods: + groups[name] = args, result, AS3 + + res = {} + attrs = dict(resolver=OverloadingResolver) + for name, methlist in groups.iteritems(): + meths = [meth(Meth([self.resolve_class(arg) for arg in args], + self.resolve_class(result))) for (args, result) in methlist] + res[name] = overload(*meths, **attrs) + return res + +from mech.fusion.avm2.library import get_playerglobal + +playerglobal_lib = get_playerglobal(Library=PyPyLibrary) +playerglobal_lib.install_global(__name__.rpartition(".")[0]+".playerglobal") Modified: pypy/branch/avm/pypy/translator/avm2/metavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/metavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/metavm.py Sat Apr 3 01:46:31 2010 @@ -1,7 +1,7 @@ from pypy.rpython.ootypesystem import ootype -from pypy.translator.oosupport.metavm import MicroInstruction, \ - PushAllArgs, StoreResult, GetField, SetField, DownCast -from pypy.translator.oosupport.metavm import _Call as _OOCall +from pypy.translator.oosupport.metavm import MicroInstruction, PushAllArgs, \ + StoreResult, GetField, SetField, DownCast, _Call as _OOCall, \ + get_primitive_name from pypy.translator.avm2.runtime import _static_meth, NativeInstance from pypy.translator.avm2 import types_ as types from mech.fusion.avm2 import constants @@ -16,13 +16,12 @@ return ret_type, arg_types class _Call(_OOCall): - def render(self, generator, op): callee = op.args[0].value if isinstance(callee, _static_meth): self._render_static_function(generator, callee, op.args) else: - _OOCall.render(self, generator, op) + generator.call_graph(op.args[0].value.graph, op.args[1:]) def _render_static_function(self, generator, funcdesc, args): import pdb; pdb.set_trace() @@ -43,6 +42,18 @@ class _CallMethod(_Call): + DISPATCH = { + ootype.Array : { + "ll_setitem_fast": lambda gen: gen.array_setitem(), + "ll_getitem_fast": lambda gen: gen.array_getitem(), + "ll_length": lambda gen: gen.get_field("length", types.types.int), + }, + ootype.AbstractString : { + "ll_append": lambda gen: gen.emit('add'), + "ll_stritem_nonneg": lambda gen: gen.call_method("charAt", 1, types.types.string), + "ll_strlen": lambda gen: gen.get_field("length", types.types.int), + }, + } def render(self, generator, op): method = op.args[0] self._render_method(generator, method.value, op.args[1:]) @@ -55,22 +66,10 @@ else: generator.load(*args) - if isinstance(this.concretetype, ootype.Array) and this.concretetype.ITEM is not ootype.Void: - if method_name == "ll_setitem_fast": - generator.array_setitem() - elif method_name == "ll_getitem_fast": - generator.array_getitem() - elif method_name == "ll_length": - generator.array_length() - else: - assert False - elif isinstance(this.concretetype, ootype.AbstractString): - if method_name == "ll_append": - generator.emit('add') - elif method_name == "ll_stritem_nonneg": - generator.call_method("charAt", 1) - elif method_name == "ll_strlen": - generator.get_field("length") + for TYPE, D in self.DISPATCH.iteritems(): + if isinstance(this.concretetype, TYPE): + D[method_name](generator) + break else: generator.call_method(method_name, len(args)-1) Modified: pypy/branch/avm/pypy/translator/avm2/runtime.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/runtime.py (original) +++ pypy/branch/avm/pypy/translator/avm2/runtime.py Sat Apr 3 01:46:31 2010 @@ -14,7 +14,6 @@ from pypy.rpython.ootypesystem.rootype import OOInstanceRepr from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.ootype import Meth, StaticMethod -# from pypy.translator.avm2 import constants ## Annotation model @@ -293,25 +292,6 @@ meth = self._static_methods[meth_name] return meth._resolver.annotate(args_s) - # def _load_class(self): - # names = self._INSTANCE._namespace.split('.') - # names.append(self._INSTANCE._classname) - # obj = PythonNet - # for name in names: - # obj = getattr(obj, name) - # self._PythonNet_class = obj - - # def __getattr__(self, attr): - # if attr in self._static_methods or attr in self._static_fields: - # self._load_class() - # return getattr(self._PythonNet_class, attr) - # else: - # raise AttributeError, attr - - # def __call__(self, *args): - # self._load_class() - # return self._PythonNet_class(*args) - class Entry(ExtRegistryEntry): _type_ = NativeClass @@ -323,190 +303,6 @@ return SomeOOInstance(self.instance._INSTANCE) -# BOXABLE_TYPES = [ootype.Signed, ootype.Unsigned, ootype.SignedLongLong, -# ootype.UnsignedLongLong, ootype.Bool, ootype.Float, -# ootype.Char, ootype.String] - -# class BoxedSpace: -# objects = {} -# index = 0 -# def put(cls, obj): -# index = cls.index -# cls.objects[index] = obj -# cls.index += 1 -# return index -# put = classmethod(put) - -# def get(cls, index): -# return cls.objects[index] -# get = classmethod(get) - -# def box(x): -# t = type(x) -# if t is int: -# return CLR.System.Int32(x) -# elif t is r_uint: -# return CLR.System.UInt32(x) -# elif t is r_longlong: -# return CLR.System.Int64(x) -# elif t is r_ulonglong: -# return CLR.System.UInt64(x) -# elif t is bool: -# return CLR.System.Boolean(x) -# elif t is float: -# return CLR.System.Double(x) -# elif t is str or t is unicode: -# if len(x) == 1: -# return CLR.System.Char(x) -# else: -# return CLR.System.String(x) -# elif isinstance(x, ootype._class): -# if hasattr(x, '_FUNC'): -# TYPE = x._FUNC -# assert isinstance(TYPE, ootype.StaticMethod) -# return typeof(TYPE) -# else: -# name = '%s.%s' % (x._INSTANCE._namespace, x._INSTANCE._classname) -# t = CLR.System.Type.GetType(name) -# assert t is not None -# return t -# elif isinstance(x, PythonNet.System.Object): -# return x -# elif x is None: -# return None -# else: -# # cast RPython instances to System.Object is trivial when -# # translated but not when interpreting, because Python for -# # .NET doesn't support passing aribrary Python objects to -# # .NET. To solve, we store them in the BoxedSpace, then we -# # return an opaque objects, which will be used by unbox to -# # retrieve the original RPython instance. -# index = BoxedSpace.put(x) -# res = PythonNet.pypy.test.ObjectWrapper(index) -# return res - -# def unbox(x, TYPE): -# if isinstance(x, PythonNet.pypy.test.ObjectWrapper): -# x = BoxedSpace.get(x.index) - -# if isinstance(TYPE, (type, types.ClassType)): -# # we need to check the TYPE and return None if it fails -# if isinstance(x, TYPE): -# return x -# else: -# return None - -# if isinstance(TYPE, ootype.OOType) and TYPE is not ootype.String and not isinstance(TYPE, ootype.StaticMethod): -# try: -# return ootype.enforce(TYPE, x) -# except TypeError: -# return None - -# # TODO: do the typechecking also in the other cases - -# # this is a workaround against a pythonnet limitation: you can't -# # directly get the, e.g., python int from the System.Int32 object: -# # a simple way to do this is to put it into an ArrayList and -# # retrieve the value. -# tmp = PythonNet.System.Collections.ArrayList() -# tmp.Add(x) -# return tmp[0] - - -# class Entry(ExtRegistryEntry): -# _about_ = box - -# def compute_result_annotation(self, x_s): -# can_be_None = getattr(x_s, 'can_be_None', False) -# return SomeOOInstance(CLR.System.Object._INSTANCE, can_be_None=can_be_None) - -# def specialize_call(self, hop): -# v_obj, = hop.inputargs(*hop.args_r) - -# hop.exception_cannot_occur() -# TYPE = v_obj.concretetype -# if (TYPE is ootype.String or isinstance(TYPE, (ootype.OOType, NativeInstance))): -# return hop.genop('ooupcast', [v_obj], hop.r_result.lowleveltype) -# else: -# if TYPE not in BOXABLE_TYPES: -# raise TyperError, "Can't box values of type %s" % v_obj.concretetype -# return hop.genop('clibox', [v_obj], hop.r_result.lowleveltype) - - -# class Entry(ExtRegistryEntry): -# _about_ = unbox - -# def compute_result_annotation(self, x_s, type_s): -# assert isinstance(x_s, SomeOOInstance) -# assert isinstance(x_s.ootype, NativeInstance) -# assert type_s.is_constant() -# TYPE = type_s.const -# if isinstance(TYPE, (type, types.ClassType)): -# # it's a user-defined class, so we return SomeInstance -# # can_be_None == True because it can always return None, if it fails -# classdef = self.bookkeeper.getuniqueclassdef(TYPE) -# return SomeInstance(classdef, can_be_None=True) -# elif TYPE in BOXABLE_TYPES: -# return OverloadingResolver.lltype_to_annotation(TYPE) -# elif isinstance(TYPE, ootype.StaticMethod): -# return SomeOOStaticMeth(TYPE) -# elif isinstance(TYPE, ootype.OOType): -# return SomeOOInstance(TYPE) -# else: -# assert False - - -# def specialize_call(self, hop): -# assert hop.args_s[1].is_constant() -# TYPE = hop.args_s[1].const -# v_obj = hop.inputarg(hop.args_r[0], arg=0) -# if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)) or isinstance(TYPE, ootype.OOType): -# return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) -# else: -# c_type = hop.inputconst(ootype.Void, TYPE) -# return hop.genop('cliunbox', [v_obj, c_type], hop.r_result.lowleveltype) - - - -# native_exc_cache = {} -# def NativeException(cliClass): -# try: -# return native_exc_cache[cliClass._name] -# except KeyError: -# res = _create_NativeException(cliClass) -# native_exc_cache[cliClass._name] = res -# return res - -# def _create_NativeException(cliClass): -# from pypy.translator.cli.support import getattr_ex -# TYPE = cliClass._INSTANCE -# if PythonNet.__name__ in ('CLR', 'clr'): -# # we are using pythonnet -- use the .NET class -# name = '%s.%s' % (TYPE._namespace, TYPE._classname) -# res = getattr_ex(PythonNet, name) -# else: -# # we are not using pythonnet -- create a fake class -# res = types.ClassType(TYPE._classname, (Exception,), {}) -# res._rpython_hints = {'NATIVE_INSTANCE': TYPE} -# return res - -# def native_exc(exc): -# return exc - -# class Entry(ExtRegistryEntry): -# _about_ = native_exc - -# def compute_result_annotation(self, exc_s): -# assert isinstance(exc_s, SomeInstance) -# cls = exc_s.classdef.classdesc.pyobj -# assert issubclass(cls, Exception) -# NATIVE_INSTANCE = cls._rpython_hints['NATIVE_INSTANCE'] -# return SomeOOInstance(NATIVE_INSTANCE) - -# def specialize_call(self, hop): -# v_obj, = hop.inputargs(*hop.args_r) -# return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype) - def new_array(type, length): return [None] * length @@ -613,69 +409,69 @@ # return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype) -def clidowncast(obj, TYPE): - return obj +## def clidowncast(obj, TYPE): +## return obj -class Entry(ExtRegistryEntry): - _about_ = clidowncast - - def compute_result_annotation(self, s_value, s_type): - if isinstance(s_type.const, ootype.OOType): - TYPE = s_type.const - else: - Class = s_type.const - TYPE = Class._INSTANCE - assert ootype.isSubclass(TYPE, s_value.ootype) - return SomeOOInstance(TYPE) +## class Entry(ExtRegistryEntry): +## _about_ = clidowncast - def specialize_call(self, hop): - assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) - v_inst = hop.inputarg(hop.args_r[0], arg=0) - return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) - -def cast_record_to_object(record): - T = ootype.typeOf(record) - assert isinstance(T, ootype.Record) - return ootype._view(playerglobal.Object._INSTANCE, record) - -def cast_object_to_record(T, obj): - assert isinstance(T, ootype.Record) - assert isinstance(obj, ootype._view) - assert isinstance(obj._inst, ootype._record) - record = obj._inst - assert ootype.typeOf(record) == T - return record - -class Entry(ExtRegistryEntry): - _about_ = cast_record_to_object - - def compute_result_annotation(self, s_value): - T = s_value.ootype - assert isinstance(T, ootype.Record) - can_be_None = getattr(s_value, 'can_be_None', False) - return SomeOOInstance(playerglobal.Object._INSTANCE, can_be_None=can_be_None) - - def specialize_call(self, hop): - assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) - v_obj, = hop.inputargs(*hop.args_r) - hop.exception_cannot_occur() - return hop.genop('ooupcast', [v_obj], hop.r_result.lowleveltype) - -class Entry(ExtRegistryEntry): - _about_ = cast_object_to_record - - def compute_result_annotation(self, s_type, s_value): - assert s_type.is_constant() - T = s_type.const - assert isinstance(T, ootype.Record) - can_be_None = getattr(s_value, 'can_be_None', False) - return SomeOOInstance(T, can_be_None) - - def specialize_call(self, hop): - assert hop.args_s[0].is_constant() - TYPE = hop.args_s[0].const - v_obj = hop.inputarg(hop.args_r[1], arg=1) - return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) +## def compute_result_annotation(self, s_value, s_type): +## if isinstance(s_type.const, ootype.OOType): +## TYPE = s_type.const +## else: +## Class = s_type.const +## TYPE = Class._INSTANCE +## assert ootype.isSubclass(TYPE, s_value.ootype) +## return SomeOOInstance(TYPE) + +## def specialize_call(self, hop): +## assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) +## v_inst = hop.inputarg(hop.args_r[0], arg=0) +## return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + +## def cast_record_to_object(record): +## T = ootype.typeOf(record) +## assert isinstance(T, ootype.Record) +## return ootype._view(playerglobal.Object._INSTANCE, record) + +## def cast_object_to_record(T, obj): +## assert isinstance(T, ootype.Record) +## assert isinstance(obj, ootype._view) +## assert isinstance(obj._inst, ootype._record) +## record = obj._inst +## assert ootype.typeOf(record) == T +## return record + +## class Entry(ExtRegistryEntry): +## _about_ = cast_record_to_object + +## def compute_result_annotation(self, s_value): +## T = s_value.ootype +## assert isinstance(T, ootype.Record) +## can_be_None = getattr(s_value, 'can_be_None', False) +## return SomeOOInstance(playerglobal.Object._INSTANCE, can_be_None=can_be_None) + +## def specialize_call(self, hop): +## assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) +## v_obj, = hop.inputargs(*hop.args_r) +## hop.exception_cannot_occur() +## return hop.genop('ooupcast', [v_obj], hop.r_result.lowleveltype) + +## class Entry(ExtRegistryEntry): +## _about_ = cast_object_to_record + +## def compute_result_annotation(self, s_type, s_value): +## assert s_type.is_constant() +## T = s_type.const +## assert isinstance(T, ootype.Record) +## can_be_None = getattr(s_value, 'can_be_None', False) +## return SomeOOInstance(T, can_be_None) + +## def specialize_call(self, hop): +## assert hop.args_s[0].is_constant() +## TYPE = hop.args_s[0].const +## v_obj = hop.inputarg(hop.args_r[1], arg=1) +## return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) #class _fieldinfo(object): # def __init__(self, llvalue): Modified: pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/entrypoint.py Sat Apr 3 01:46:31 2010 @@ -1,5 +1,5 @@ - import py +from pypy.conftest import option from pypy.translator.avm2.test.browsertest import browsertest from pypy.translator.avm2.avm2gen import PyPyAvm2ilasm @@ -8,6 +8,7 @@ DefineEditText, SymbolClass, PlaceObject2, DoABC, ShowFrame, End from mech.fusion.swf.records import Rect, RGBA +from mech.fusion.avm2.abc_ import AbcFile from mech.fusion.avm2.constants import QName, packagedQName from mech.fusion.avm2.traits import AbcSlotTrait @@ -25,8 +26,8 @@ '''.strip() % (name,)), color=RGBA(0xFFFFFF))) self.swf.add_tag(PlaceObject2(1, 2, name="edittext")) self.abc = DoABC("PyPy Main") - self.actions = PyPyAvm2ilasm(gen.db, self.abc) - + self.actions = PyPyAvm2ilasm(gen.db, self.abc, option.mf_optim) + self.swf.add_tag(self.abc) self.swf.add_tag(SymbolClass({0:"PyPyTest_EntryPoint"})) self.swf.add_tag(ShowFrame()) @@ -40,9 +41,6 @@ self.actions.store_var('text') self.update_text() - def serialize_swf(self): - return self.swf.serialize() - def print_var(self, prefix, varname): self.actions.push_const(prefix) self.actions.push_var(varname) @@ -58,12 +56,12 @@ def start_test_maincls(self): pass - + def start_test(self): self.start_test_maincls() if self.excwrap: self.actions.begin_try() - + def finish_test(self): # WHHEEEEEEE! if self.excwrap: @@ -83,7 +81,7 @@ def do_test(self): pass - + def epilogue(self): pass @@ -97,12 +95,13 @@ f.write(self.swf.serialize()) f.close() f = open("%s.flash.abc" % (self.name,), "wb") - f.write(self.abc.serialize()) + f.write(AbcFile.serialize(self.abc)) f.close() + return browsertest(self.name, self.swf) def get_edittext(self): - if not self.actions.HL('edittext'): + if not self.actions.HL('edittext'): self.actions.push_this() self.actions.get_field('edittext') self.actions.store_var('edittext') @@ -148,16 +147,17 @@ class TamarinTestEntryPoint(BaseTestEntryPoint): def do_test(self): + self.finish_test() f = open("%s.tamarin.abc" % (self.name,), "wb") - f.write(self.abc.serialize()) + f.write(AbcFile.serialize(self.abc)) f.close() asdf def update_text(self): + self.actions.emit('findpropstrict', QName("print")) self.actions.push_const("\n") - self.actions.get_field('text') + self.actions.push_var('text') self.actions.emit('add') - self.actions.emit('findpropstrict', QName("print")) self.actions.emit('callpropvoid', QName("print"), 1) def epilogue(self): Modified: pypy/branch/avm/pypy/translator/avm2/test/runtest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/runtest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/runtest.py Sat Apr 3 01:46:31 2010 @@ -40,33 +40,25 @@ pass return string -def compile_function(func, name, annotation=[], graph=None, backendopt=True, +def compile_function(func, name, annotation=[], backendopt=True, exctrans=False, annotatorpolicy=None, wrapexc=False): olddefs = patch_os() - gen = _build_gen(func, annotation, graph, backendopt, - exctrans, annotatorpolicy) + gen = _build_gen(func, annotation, backendopt, exctrans, annotatorpolicy) entry_point = ENTRY_POINTS[option.tamtarget](name, gen, wrapexc) gen.ilasm = entry_point.actions gen.generate_source() unpatch_os(olddefs) # restore original values - return entry_point + return entry_point, gen -def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, +def _build_gen(func, annotation, backendopt=True, exctrans=False, annotatorpolicy=None): - try: + try: func = func.im_func - except AttributeError: + except AttributeError: pass t = TranslationContext() - if graph is not None: - graph.func = func - ann = t.buildannotator(policy=annotatorpolicy) - inputcells = [ann.typeannotation(an) for an in annotation] - ann.build_graph_types(graph, inputcells) - t.graphs.insert(0, graph) - else: - ann = t.buildannotator(policy=annotatorpolicy) - ann.build_types(func, annotation) + ann = t.buildannotator(policy=annotatorpolicy) + ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() if backendopt: @@ -76,9 +68,8 @@ if option.view: t.view() - main_graph = t.graphs[0] tmpdir = py.path.local('.') - + return GenAVM2(tmpdir, t, None, exctrans) class AVM2Test(BaseRtypingTest, OORtypeMixin): @@ -88,7 +79,6 @@ self._harness = None def _compile(self, fn, args, ann=None, backendopt=True, exctrans=False, wrapexc=False): - frame = sys._getframe() while frame: name = frame.f_code.co_name @@ -97,10 +87,10 @@ frame = frame.f_back else: name = "test_unknown" - + if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] - + self._entry_point = compile_function(fn, "%s.%s" % (type(self).__name__, name), ann, backendopt=backendopt, exctrans=exctrans, wrapexc=wrapexc) self._func = fn @@ -117,25 +107,26 @@ def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): pass - + def _get_backendopt(self, backendopt): if backendopt is None: backendopt = getattr(self, 'backendopt', True) # enable it by default return backendopt - - def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False, wrapexc=False): + + def interpret(self, fn, args, annotation=None, backendopt=None, + exctrans=False, wrapexc=False): backendopt = self._get_backendopt(backendopt) - entry_point = self._compile(fn, args, annotation, backendopt=backendopt, - exctrans=exctrans, wrapexc=wrapexc) - print entry_point + entry_point, gen = self._compile(fn, args, annotation, + backendopt, exctrans, wrapexc) entry_point.start_test() - entry_point.actions.call_function_constargs(fn.func_name, *args) + entry_point.actions.call_graph(gen.translator.graphs[0], args) result = parse_result(entry_point.do_test()) if isinstance(result, ExceptionWrapper): raise result return result - - def interpret_raises(self, exception, fn, args, backendopt=None, exctrans=False): + + def interpret_raises(self, exception, fn, args, backendopt=None, + exctrans=False): import exceptions # needed by eval backendopt = self._get_backendopt(backendopt) try: @@ -163,7 +154,7 @@ return t def class_name(self, value): - return value.class_name.split(".")[-1] + return value.class_name.split(".")[-1] def is_of_instance_type(self, val): return isinstance(val, InstanceWrapper) Modified: pypy/branch/avm/pypy/translator/avm2/types_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/types_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/types_.py Sat Apr 3 01:46:31 2010 @@ -1,5 +1,5 @@ """ -Translate between PyPy ootypesystem and .NET Common Type System +bTranslate between PyPy ootypesystem and .NET Common Type System """ import exceptions @@ -22,74 +22,75 @@ def typename(self): raise NotImplementedError + def load_default(self, asm): + raise NotImplementedError + def __str__(self): - return self.typename() + return str(self.multiname()) def __hash__(self): - return hash(self.typename()) - + return hash(self.multiname()) + def __eq__(self, other): - return isinstance(other, self.__class__) and self.typename() == other.typename() - - def __ne__(self, other): - return self.typename() != other.typename() + return isinstance(other, type(self)) and self.multiname() == other.multiname() + def __ne__(self, other): + return not self == other class Avm2PrimitiveType(Avm2Type): - def __init__(self, name): - self.name = name + def __init__(self, name, default): + self.name, self.default = name, default + + def load_default(self, gen): + gen.load(self.default) def typename(self): return self.name def multiname(self): return constants.QName(self.typename()) - class Avm2NamespacedType(Avm2Type): nstype = constants.TYPE_NAMESPACE_PackageNamespace - def __init__(self, name, namespace=''): - print "Namedspaced Type", name if '::' in name and namespace == '': self.ns, self.name = name.rsplit('::', 1) else: self.name = name self.ns = namespace - + def typename(self): return "%s::%s" % (self.ns, self.name) - - def classname(self): - raise NotImplementedError def multiname(self): return constants.QName(self.name, constants.Namespace(self.nstype, self.ns)) - class Avm2ArrayType(Avm2Type): - def __init__(self, itemtype): - self.itemtype = itemtype + self.ITEM = itemtype + + def load_default(self, gen): + gen.oonewarray(self, 0) + + def __str__(self): + return "%s.<%s>" % (vec_qname, constants.QName(self.ITEM)) def multiname(self): - return constants.TypeName(vec_qname, self.itemtype.multiname()) + return constants.TypeName(vec_qname, constants.QName(self.ITEM)) T = Avm2PrimitiveType -N = Avm2NamespacedType +# N = Avm2NamespacedType class types: - void = T('void') - int = T('int') - uint = T('uint') - bool = T('Boolean') - float = T('Number') - string = T('String') - - # weakref = CliClassType('pypylib', 'pypy.runtime.WeakReference') - type = T('Class') - object = T('Object') + void = T('void', constants.null) + int = T('int', -1) + uint = T('uint', 0) + bool = T('Boolean', False) + float = T('Number', float('nan')) + string = T('String', "") + type = T('Class', constants.QName('Class')) + object = T('Object', {}) list = Avm2ArrayType - dict = T('Object') + dict = T('Object', {}) # sb = N('StringBuilder', 'pypy.lib') del T @@ -104,9 +105,9 @@ ootype.Char: types.string, ootype.UniChar: types.string, ootype.Class: types.type, -# ootype.String: types.string, + ootype.String: types.string, # ootype.StringBuilder: types.sb, -# ootype.Unicode: types.string, + ootype.Unicode: types.string, # ootype.UnicodeBuilder: types.sb, # maps generic types to their ordinal @@ -114,24 +115,14 @@ ootype.Dict.SELFTYPE_T: types.dict, } - -def _get_from_dict(d, key, error): - try: - return d[key] - except KeyError: - if getoption('nostop'): - log.WARNING(error) - return key - else: - assert False, error - class Avm2TypeSystem(object): - def __init__(self, db): self.db = db def lltype_to_cts(self, t): - if t is ootype.ROOT: + if isinstance(t, Avm2Type): + return t + elif t is ootype.ROOT: return types.object elif isinstance(t, lltype.Ptr) and isinstance(t.TO, lltype.OpaqueType): return types.object @@ -155,16 +146,16 @@ key_type = self.lltype_to_cts(t._KEYTYPE) value_type = self.lltype_to_cts(t._VALUETYPE) return types.dict - elif isinstance(t, ootype.DictItemsIterator): - key_type = self.lltype_to_cts(t._KEYTYPE) - value_type = self.lltype_to_cts(t._VALUETYPE) - if key_type == types.void: - key_type = types.int32 # placeholder - if value_type == types.void: - value_type = types.int32 # placeholder - return types.dict_items_iterator.specialize(key_type, value_type) + ## elif isinstance(t, ootype.DictItemsIterator): + ## key_type = self.lltype_to_cts(t._KEYTYPE) + ## value_type = self.lltype_to_cts(t._VALUETYPE) + ## if key_type == types.void: + ## key_type = types.int32 # placeholder + ## if value_type == types.void: + ## value_type = types.int32 # placeholder + ## return types.dict_items_iterator.specialize(key_type, value_type) - return _get_from_dict(_lltype_to_cts, t, 'Unknown type %s' % t) + return _lltype_to_cts.get(t, None) def llvar_to_cts(self, var): return self.lltype_to_cts(var.concretetype), var.name @@ -176,13 +167,13 @@ return name def graph_to_qname(self, graph): - func_name = self.graph.name - namespace = getattr(self.graph, '_namespace_', None) + func_name = graph.name + namespace = getattr(graph, '_namespace_', None) if namespace: return constants.packagedQName(namespace, func_name) else: return constants.QName(func_name) - + # def ctor_name(self, t): # return 'instance void %s::.ctor()' % self.lltype_to_cts(t) Modified: pypy/branch/avm/pypy/translator/driver.py ============================================================================== --- pypy/branch/avm/pypy/translator/driver.py (original) +++ pypy/branch/avm/pypy/translator/driver.py Sat Apr 3 01:46:31 2010 @@ -9,7 +9,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings from pypy.annotation import policy as annpolicy -from py.compat import optparse +import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS @@ -546,6 +546,28 @@ [STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE], "LLInterpreting") + def task_compile_tamarin(self): + from pypy.translator.avm2.genavm import GenAVM2 + from pypy.translator.avm2.entrypoint import get_entrypoint + + if self.entry_point is not None: # executable mode + entry_point_graph = self.translator.graphs[0] + entry_point = get_entrypoint(entry_point_graph) + else: + # library mode + assert self.libdef is not None + bk = self.translator.annotator.bookkeeper + entry_point = self.libdef.get_entrypoint(bk) + + self.gen = GenAVM2(udir, self.translator, entry_point, config=self.config) + self.gen.generate_source() + abc = self.gen.serialize_abc() + f = open(entry_point.get_name() + ".abc", "wb") + f.write(abc) + f.close() + task_compile_tamarin = taskdef(task_compile_tamarin, ["?" + OOBACKENDOPT, OOTYPE], + 'Generating ABC bytecode') + def task_source_cli(self): from pypy.translator.cli.gencli import GenCli from pypy.translator.cli.entrypoint import get_entrypoint @@ -599,14 +621,25 @@ shutil.copy(os.path.join(usession_path, 'main.il'), dirname) newexename = basename f = file(newexename, 'w') - f.write("""#!/bin/bash + f.write(r"""#!/bin/bash LEDIT=`type -p ledit` EXE=`readlink $0` if [ -z $EXE ] then EXE=$0 fi -if uname -s | grep -iq Cygwin ; then MONO=; else MONO=mono; fi +if uname -s | grep -iq Cygwin +then + MONO= +else + MONO=mono + # workaround for known mono buggy versions + VER=`mono -V | head -1 | sed s/'Mono JIT compiler version \(.*\) (.*'/'\1/'` + if [[ 2.1 < "$VER" && "$VER" < 2.4.3 ]] + then + MONO="mono -O=-branch" + fi +fi $LEDIT $MONO "$(dirname $EXE)/$(basename $EXE)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() Modified: pypy/branch/avm/pypy/translator/interactive.py ============================================================================== --- pypy/branch/avm/pypy/translator/interactive.py (original) +++ pypy/branch/avm/pypy/translator/interactive.py Sat Apr 3 01:46:31 2010 @@ -1,4 +1,4 @@ -from py.compat import optparse +import optparse import autopath from pypy.translator.translator import TranslationContext @@ -150,7 +150,13 @@ backend = self.ensure_backend() getattr(self.driver, 'compile_'+backend)() return self.driver.c_entryp - + + def compile_tamarin(self, argtypes=None, **kwds): + self.update_options(argtypes, kwds) + self.ensure_backend('tamarin') + self.driver.compile_tamarin() + return self.driver.c_entryp + def compile_c(self, argtypes=None, **kwds): self.update_options(argtypes, kwds) self.ensure_backend('c') Modified: pypy/branch/avm/pypy/translator/tool/autopath.py ============================================================================== --- pypy/branch/avm/pypy/translator/tool/autopath.py (original) +++ pypy/branch/avm/pypy/translator/tool/autopath.py Sat Apr 3 01:46:31 2010 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break From fijal at codespeak.net Sat Apr 3 01:48:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:48:28 +0200 (CEST) Subject: [pypy-svn] r73316 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402234828.EC415282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:48:26 2010 New Revision: 73316 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: implement _PyString_Resize Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:48:26 2010 @@ -2,12 +2,16 @@ from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, build_type_checkers, - PyObjectP) + PyObjectP, cpython_api_c) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") + at cpython_api_c() +def PyString_FromFormatV(): + pass + @cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): if char_p: @@ -50,8 +54,8 @@ w_obj = from_ref(space, ref) return space.int_w(space.len(w_obj)) - at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyString_Resize(space, w_string, newsize): + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=CANNOT_FAIL) +def _PyString_Resize(space, ref, newsize): """A way to resize a string object even though it is "immutable". Only use this to build up a brand new string object; don't use this if the string may already be known in other parts of the code. It is an error to call this function if the @@ -64,5 +68,8 @@ This function used an int type for newsize. This might require changes in your code for properly supporting 64-bit systems.""" - import pdb - pdb.set_trace() + # XXX always create a new string so fa + w_s = from_ref(space, ref[0]) + ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) + ref[0] = ptr + return 0 From fijal at codespeak.net Sat Apr 3 01:52:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:52:33 +0200 (CEST) Subject: [pypy-svn] r73317 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402235233.E2781282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:52:32 2010 New Revision: 73317 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: decref the old string Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:52:32 2010 @@ -3,7 +3,7 @@ PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, build_type_checkers, PyObjectP, cpython_api_c) -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") @@ -71,5 +71,6 @@ # XXX always create a new string so fa w_s = from_ref(space, ref[0]) ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) + Py_DecRef(space, ref[0]) ref[0] = ptr return 0 From fijal at codespeak.net Sat Apr 3 01:53:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:53:45 +0200 (CEST) Subject: [pypy-svn] r73318 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402235345.B88B0282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:53:43 2010 New Revision: 73318 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: error=-1 here Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:53:43 2010 @@ -54,7 +54,7 @@ w_obj = from_ref(space, ref) return space.int_w(space.len(w_obj)) - at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=CANNOT_FAIL) + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) def _PyString_Resize(space, ref, newsize): """A way to resize a string object even though it is "immutable". Only use this to build up a brand new string object; don't use this if the string may already be From fijal at codespeak.net Sat Apr 3 01:55:05 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:55:05 +0200 (CEST) Subject: [pypy-svn] r73319 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402235505.0B65E282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:55:03 2010 New Revision: 73319 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: typo Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:55:03 2010 @@ -68,7 +68,7 @@ This function used an int type for newsize. This might require changes in your code for properly supporting 64-bit systems.""" - # XXX always create a new string so fa + # XXX always create a new string so far w_s = from_ref(space, ref[0]) ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) Py_DecRef(space, ref[0]) From fijal at codespeak.net Sat Apr 3 01:59:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 01:59:48 +0200 (CEST) Subject: [pypy-svn] r73320 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100402235948.BAFCB282B90@codespeak.net> Author: fijal Date: Sat Apr 3 01:59:47 2010 New Revision: 73320 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: make it do exactly what docstring says Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 3 01:59:47 2010 @@ -70,7 +70,12 @@ require changes in your code for properly supporting 64-bit systems.""" # XXX always create a new string so far w_s = from_ref(space, ref[0]) - ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) + try: + ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) + except: + Py_DecRef(space, ref[0]) + ref[0] = lltype.nullptr(PyStringObject) + raise Py_DecRef(space, ref[0]) ref[0] = ptr return 0 From fijal at codespeak.net Sat Apr 3 02:04:03 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Apr 2010 02:04:03 +0200 (CEST) Subject: [pypy-svn] r73321 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403000403.7130D282B90@codespeak.net> Author: fijal Date: Sat Apr 3 02:04:02 2010 New Revision: 73321 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: this is actually flags Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Sat Apr 3 02:04:02 2010 @@ -14,7 +14,7 @@ @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=0) -def pypy_vgetargs1(space, w_obj, fmt, va_list_p, lgt): +def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): i = 0 while True: c = fmt[i] From trundle at codespeak.net Sat Apr 3 02:59:36 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Sat, 3 Apr 2010 02:59:36 +0200 (CEST) Subject: [pypy-svn] r73322 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100403005936.1EDA6282B90@codespeak.net> Author: trundle Date: Sat Apr 3 02:59:34 2010 New Revision: 73322 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Add T_STRING and T_STRING_INPLACE members. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Sat Apr 3 02:59:34 2010 @@ -22,7 +22,9 @@ /* Types */ #define T_INT 1 +#define T_STRING 5 #define T_OBJECT 6 +#define T_STRING_INPLACE 13 /* Strings contained in the structure */ #define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError when the value is NULL, instead of converting to None. */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Sat Apr 3 02:59:34 2010 @@ -6,6 +6,7 @@ from pypy.module.cpyext.intobject import PyInt_AsLong from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, from_ref, make_ref +from pypy.module.cpyext.stringobject import PyString_FromString from pypy.module.cpyext.typeobjectdefs import PyMemberDef @@ -17,6 +18,15 @@ if member_type == structmemberdefs.T_INT: result = rffi.cast(rffi.INTP, addr) w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_STRING: + result = rffi.cast(rffi.CCHARPP, addr) + if result[0]: + w_result = PyString_FromString(space, result[0]) + else: + w_result = space.w_None + elif member_type == structmemberdefs.T_STRING_INPLACE: + result = rffi.cast(rffi.CCHARP, addr) + w_result = PyString_FromString(space, result) elif member_type in [structmemberdefs.T_OBJECT, structmemberdefs.T_OBJECT_EX]: obj_ptr = rffi.cast(PyObjectP, addr) @@ -40,7 +50,9 @@ member_type = rffi.cast(lltype.Signed, w_member.c_type) flags = rffi.cast(lltype.Signed, w_member.c_flags) - if flags & structmemberdefs.READONLY: + if (flags & structmemberdefs.READONLY or + member_type in [structmemberdefs.T_STRING, + structmemberdefs.T_STRING_INPLACE]): raise OperationError(space.w_TypeError, space.wrap("readonly attribute")) elif w_value is None: @@ -51,7 +63,7 @@ elif member_type != structmemberdefs.T_OBJECT: raise OperationError(space.w_TypeError, space.wrap("can't delete numeric/char attribute")) - + if member_type == structmemberdefs.T_INT: w_long_value = PyInt_AsLong(space, w_value) array = rffi.cast(rffi.INTP, addr) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Sat Apr 3 02:59:34 2010 @@ -1,5 +1,7 @@ T_INT = 1 +T_STRING = 5 T_OBJECT = 6 +T_STRING_INPLACE = 13 T_OBJECT_EX = 16 READONLY = RO = 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Sat Apr 3 02:59:34 2010 @@ -5,6 +5,8 @@ PyObject_HEAD int foo; /* the context holder */ PyObject *foo_object; + char *foo_string; + char foo_string_inplace[5]; } fooobject; static PyTypeObject footype; @@ -20,6 +22,8 @@ foop->foo = 42; foop->foo_object = NULL; + foop->foo_string = "Hello from PyPy"; + strncpy(foop->foo_string_inplace, "spam", 5); return foop; } @@ -48,9 +52,17 @@ return (PyObject *)foop; } +static PyObject * +foo_unset(fooobject *self) +{ + self->foo_string = NULL; + Py_RETURN_NONE; +} + static PyMethodDef foo_methods[] = { {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, + {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; @@ -105,6 +117,10 @@ "A Python object."}, {"object_member_ex", T_OBJECT_EX, offsetof(fooobject, foo_object), 0, "A Python object."}, + {"string_member", T_STRING, offsetof(fooobject, foo_string), 0, + "A string."}, + {"string_member_inplace", T_STRING_INPLACE, + offsetof(fooobject, foo_string_inplace), 0, "An inplace string."}, {NULL} /* Sentinel */ }; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sat Apr 3 02:59:34 2010 @@ -54,6 +54,15 @@ del obj.object_member_ex raises(AttributeError, "del obj.object_member_ex") + assert obj.string_member == "Hello from PyPy" + raises(TypeError, "obj.string_member = 42") + raises(TypeError, "del obj.string_member") + obj.unset_string_member() + assert obj.string_member is None + assert obj.string_member_inplace == "spam" + raises(TypeError, "obj.string_member_inplace = 42") + raises(TypeError, "del obj.string_member_inplace") + a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message skip("In Progress") From trundle at codespeak.net Sat Apr 3 03:23:28 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Sat, 3 Apr 2010 03:23:28 +0200 (CEST) Subject: [pypy-svn] r73323 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100403012328.81E06282B90@codespeak.net> Author: trundle Date: Sat Apr 3 03:23:26 2010 New Revision: 73323 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Test and fix docstrings of members. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Sat Apr 3 03:23:26 2010 @@ -112,7 +112,7 @@ "A helpful docstring."}, {"int_member_readonly", T_INT, offsetof(fooobject, foo), READONLY, "A helpful docstring."}, - {"broken_member", 0xaffe, 0, 0, ""}, + {"broken_member", 0xaffe, 0, 0, NULL}, {"object_member", T_OBJECT, offsetof(fooobject, foo_object), 0, "A Python object."}, {"object_member_ex", T_OBJECT_EX, offsetof(fooobject, foo_object), 0, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sat Apr 3 03:23:26 2010 @@ -39,7 +39,8 @@ assert "readonly" in str(exc.value) raises(SystemError, "obj.broken_member") raises(SystemError, "obj.broken_member = 42") - + assert module.fooType.broken_member.__doc__ is None + assert module.fooType.object_member.__doc__ == "A Python object." assert obj.object_member is None obj.object_member = "hello" assert obj.object_member == "hello" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sat Apr 3 03:23:26 2010 @@ -59,8 +59,8 @@ self.name = rffi.charp2str(member.c_name) flags = rffi.cast(lltype.Signed, member.c_flags) doc = set = None - if doc: - doc = rffi.charp2str(getset.c_doc) + if member.c_doc: + doc = rffi.charp2str(member.c_doc) get = GettersAndSetters.member_getter.im_func del_ = GettersAndSetters.member_delete.im_func if not (flags & structmemberdefs.READONLY): From benjamin at codespeak.net Sat Apr 3 04:05:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Apr 2010 04:05:52 +0200 (CEST) Subject: [pypy-svn] r73324 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403020552.49163282B90@codespeak.net> Author: benjamin Date: Sat Apr 3 04:05:50 2010 New Revision: 73324 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: placate 64 bit platforms Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Sat Apr 3 04:05:50 2010 @@ -22,5 +22,6 @@ return 1 if c == "i": arr = api.va_get_int_star(va_list_p) - arr[0] = space.int_w(space.getitem(w_obj, space.wrap(i))) + arr[0] = rffi.cast(rffi.INT, + space.int_w(space.getitem(w_obj, space.wrap(i)))) i += 1 From xoraxax at codespeak.net Sat Apr 3 04:53:43 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 04:53:43 +0200 (CEST) Subject: [pypy-svn] r73325 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100403025343.64C40282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 04:53:40 2010 New Revision: 73325 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Extended slots logic to cope with subclassing. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 3 04:53:40 2010 @@ -372,7 +372,8 @@ def bootstrap_types(space): from pypy.module.cpyext.pyobject import make_ref - from pypy.module.cpyext.typeobject import PyTypeObjectPtr, PyPyType_Ready + from pypy.module.cpyext.typeobject import PyTypeObjectPtr, PyPyType_Ready, \ + inherit_slots # bootstrap this damn cycle type_pto = make_ref(space, space.w_type) type_pto = rffi.cast(PyTypeObjectPtr, type_pto) @@ -385,6 +386,7 @@ PyPyType_Ready(space, type_pto, space.w_type) type_pto.c_tp_bases = make_ref(space, space.newtuple([space.w_object])) object_pto.c_tp_bases = make_ref(space, space.newtuple([])) + inherit_slots(space, type_pto, space.w_object) #_____________________________________________________ # Build the bridge DLL, Allow extension DLLs to call Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sat Apr 3 04:53:40 2010 @@ -15,6 +15,7 @@ print "Obj has type", type(obj) assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) + print "type of type of obj has type", type(type(type(obj))) obj2 = obj.copy() assert module.new().name == "Foo Example" c = module.fooType.copy @@ -66,10 +67,9 @@ a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message - skip("In Progress") class bar(module.fooType): pass + assert "cannot create" in raises(TypeError, "bar()").value.message fuu = module.FuuType fuu_inst = fuu(u"abc") - assert "cannot create" in raises(TypeError, "bar()").value.message Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sat Apr 3 04:53:40 2010 @@ -18,6 +18,7 @@ Py_TPFLAGS_HAVE_CLASS from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module +from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.state import State @@ -25,11 +26,11 @@ from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ - PyGetSetDef, PyMemberDef + PyGetSetDef, PyMemberDef, newfunc from pypy.module.cpyext.slotdefs import slotdefs from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.rstring import rsplit -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -100,7 +101,7 @@ def update_all_slots(space, w_obj, pto): # XXX fill slots in pto state = space.fromcache(State) - for method_name, slot_name, slot_func, _, _, doc in slotdefs: + for method_name, slot_name, slot_func, _, _, _ in slotdefs: w_descr = space.lookup(w_obj, method_name) if w_descr is None: # XXX special case iternext @@ -109,12 +110,14 @@ if WARN_ABOUT_MISSING_SLOT_FUNCTIONS: os.write(2, method_name + " defined by the type but no slot function defined!\n") continue - if not we_are_translated() and method_name == "__new__" and "bar" in repr(w_obj): - import pdb; pdb.set_trace() slot_func_helper = llhelper(slot_func.api_func.functype, slot_func.api_func.get_wrapper(space)) - # XXX special case wrapper-functions and use a "specific" slot func, - # XXX special case tp_new + # XXX special case wrapper-functions and use a "specific" slot func + + # the special case of __new__ in CPython works a bit differently, hopefully + # this matches the semantics + if method_name == "__new__" and not pto.c_tp_new: + continue if len(slot_name) == 1: setattr(pto, slot_name[0], slot_func_helper) else: @@ -146,11 +149,38 @@ continue dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, func_voidp) + if pto.c_tp_new: + add_tp_new_wrapper(space, dict_w, pto) + + at cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) +def tp_new_wrapper(space, w_self, w_args, w_kwds): # XXX untested code + args_w = space.listview(w_args)[:] + args_w.insert(0, w_self) + w_args_new = space.newlist(args_w) + return space.call(space.lookup(space.w_type, space.wrap("__new__")), w_args_new, w_kwds) + + at specialize.memo() +def get_new_method_def(space): + from pypy.module.cpyext.modsupport import PyMethodDef + ptr = llmemory.malloc(Ptr(PyMethodDef), flavor="raw") + ptr.c_ml_name = rffi.str2charp("__new__") + ptr.c_ml_meth = llhelper(tp_new_wrapper.api_func.functype, + tp_new_wrapper.get_wrapper(space)) + ptr.c_ml_flags = METH_VARARGS | METH_KEYWORDS + ptr.c_ml_doc = rffi.str2charp("T.__new__(S, ...) -> a new object with type S, a subtype of T") + return ptr + +def add_tp_new_wrapper(space, dict_w, pto): + if "__new__" in dict_w: + return + pyo = rffi.cast(PyObject, pto) + dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space), + from_ref(space, pyo)) def inherit_special(space, pto, base_pto): - # XXX copy basicsize and flags in a magical way + # XXX missing: copy basicsize and flags in a magical way flags = rffi.cast(lltype.Signed, pto.c_tp_flags) - if flags & Py_TPFLAGS_HAVE_CLASS: + if True or flags & Py_TPFLAGS_HAVE_CLASS: # XXX i do not understand this check base_object_pyo = make_ref(space, space.w_object, steal=True) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) if base_pto != base_object_pto or \ @@ -161,7 +191,7 @@ class W_PyCTypeObject(W_TypeObject): def __init__(self, space, pto): - bases_w = [] # XXX fill + bases_w = space.fixedview(from_ref(space, pto.c_tp_bases)) dict_w = {} add_operators(space, dict_w, pto) @@ -179,7 +209,7 @@ class __extend__(W_Root): __metaclass__ = extendabletype - __slots__ = ("_pyolifeline", ) + __slots__ = ("_pyolifeline", ) # hint for the annotator _pyolifeline = None def set_pyolifeline(self, lifeline): self._pyolifeline = lifeline @@ -261,20 +291,18 @@ @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) - assert pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE base = pto this_func_ptr = llhelper(subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) - ref_of_object_type = rffi.cast(PyTypeObjectPtr, - make_ref(space, space.w_object, steal=True)) while base.c_tp_dealloc == this_func_ptr: base = base.c_tp_base assert base dealloc = base.c_tp_dealloc # XXX call tp_del if necessary generic_cpy_call(space, dealloc, obj) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) + if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) @cpython_api([PyObject], lltype.Void, external=False) @@ -302,7 +330,7 @@ obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) pto = rffi.cast(PyObject, type_pto) - Py_DecRef(space, pto) + Py_DecRef(space, pto) # XXX duplicate decref? (see above) def allocate_type_obj(space, w_type): @@ -362,6 +390,10 @@ elif pto.c_tp_base: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize + # will be filled later on with the correct value + # may not be 0 + if space.is_w(w_type, space.w_object): + pto.c_tp_new = rffi.cast(newfunc, 1) update_all_slots(space, w_type, pto) return pto @@ -369,6 +401,19 @@ def PyType_Ready(space, pto): return PyPyType_Ready(space, pto, None) +def inherit_slots(space, pto, w_base): + # XXX missing: nearly everything + base_pyo = make_ref(space, w_base) + try: + base = rffi.cast(PyTypeObjectPtr, base_pyo) + if not pto.c_tp_dealloc: + pto.c_tp_dealloc = base.c_tp_dealloc + # XXX check for correct GC flags! + if not pto.c_tp_free: + pto.c_tp_free = base.c_tp_free + finally: + Py_DecRef(space, base_pyo) + def PyPyType_Ready(space, pto, w_obj): try: pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported @@ -396,10 +441,13 @@ pto.c_tp_bases = make_ref(space, bases) if w_obj is None: PyPyType_Register(space, pto) - # missing: if base: inherit_special(space, pto, base) - # inherit_slots, setting __doc__ if not defined and tp_doc defined + if pto.c_tp_bases: # this is false while bootstrapping + for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): + inherit_slots(space, pto, w_base) + # missing: + # setting __doc__ if not defined and tp_doc defined # inheriting tp_as_* slots # unsupported: # tp_mro, tp_subclasses From xoraxax at codespeak.net Sat Apr 3 05:03:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 05:03:29 +0200 (CEST) Subject: [pypy-svn] r73326 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403030329.7C351282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 05:03:27 2010 New Revision: 73326 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Log: Add comments about pytypeobject fields. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Sat Apr 3 05:03:27 2010 @@ -152,79 +152,86 @@ ("doc", rffi.CCHARP), )) +# These fields are supported and used in different ways +# The following comments mean: +# #E essential, initialized for all PTOs +# #S supported +# #U unsupported +# #N not yet implemented PyTypeObjectFields = [] PyTypeObjectFields.extend(PyVarObjectFields) PyTypeObjectFields.extend([ - ("tp_name", rffi.CCHARP), # For printing, in format "." - ("tp_basicsize", Py_ssize_t), ("tp_itemsize", Py_ssize_t), # For allocation + ("tp_name", rffi.CCHARP), #E For printing, in format "." + ("tp_basicsize", Py_ssize_t), #E For allocation + ("tp_itemsize", Py_ssize_t), #E " # Methods to implement standard operations - ("tp_dealloc", destructor), - ("tp_print", printfunc), - ("tp_getattr", getattrfunc), - ("tp_setattr", setattrfunc), - ("tp_compare", cmpfunc), - ("tp_repr", reprfunc), + ("tp_dealloc", destructor), #E + ("tp_print", printfunc), #U + ("tp_getattr", getattrfunc), #U + ("tp_setattr", setattrfunc), #U + ("tp_compare", cmpfunc), #N + ("tp_repr", reprfunc), #N # Method suites for standard classes - ("tp_as_number", Ptr(PyNumberMethods)), - ("tp_as_sequence", Ptr(PySequenceMethods)), - ("tp_as_mapping", Ptr(PyMappingMethods)), + ("tp_as_number", Ptr(PyNumberMethods)), #N + ("tp_as_sequence", Ptr(PySequenceMethods)), #N + ("tp_as_mapping", Ptr(PyMappingMethods)), #N # More standard operations (here for binary compatibility) - ("tp_hash", hashfunc), - ("tp_call", ternaryfunc), - ("tp_str", reprfunc), - ("tp_getattro", getattrofunc), - ("tp_setattro", setattrofunc), + ("tp_hash", hashfunc), #N + ("tp_call", ternaryfunc), #N + ("tp_str", reprfunc), #N + ("tp_getattro", getattrofunc),#N + ("tp_setattro", setattrofunc),#N # Functions to access object as input/output buffer - ("tp_as_buffer", Ptr(PyBufferProcs)), + ("tp_as_buffer", Ptr(PyBufferProcs)), #U # Flags to define presence of optional/expanded features - ("tp_flags", lltype.Signed), + ("tp_flags", lltype.Signed), #E - ("tp_doc", rffi.CCHARP), # Documentation string + ("tp_doc", rffi.CCHARP), #N Documentation string # Assigned meaning in release 2.0 # call function for all accessible objects - ("tp_traverse", traverseproc), + ("tp_traverse", traverseproc),#U # delete references to contained objects - ("tp_clear", inquiry), + ("tp_clear", inquiry), #U # Assigned meaning in release 2.1 # rich comparisons - ("tp_richcompare", richcmpfunc), + ("tp_richcompare", richcmpfunc), #N # weak reference enabler - ("tp_weaklistoffset", Py_ssize_t), + ("tp_weaklistoffset", Py_ssize_t), #U # Added in release 2.2 # Iterators - ("tp_iter", getiterfunc), - ("tp_iternext", iternextfunc), + ("tp_iter", getiterfunc), #N + ("tp_iternext", iternextfunc), #N # Attribute descriptor and subclassing stuff - ("tp_methods", Ptr(PyMethodDef)), - ("tp_members", Ptr(PyMemberDef)), - ("tp_getset", Ptr(PyGetSetDef)), - ("tp_base", Ptr(PyTypeObject)), - ("tp_dict", PyObject), - ("tp_descr_get", descrgetfunc), - ("tp_descr_set", descrsetfunc), - ("tp_dictoffset", Py_ssize_t), # can be ignored in PyPy - ("tp_init", initproc), - ("tp_alloc", allocfunc), - ("tp_new", newfunc), - ("tp_free", freefunc), # Low-level free-memory routine - ("tp_is_gc", inquiry), # For PyObject_IS_GC - ("tp_bases", PyObject), - ("tp_mro", PyObject), # method resolution order - ("tp_cache", PyObject), - ("tp_subclasses", PyObject), - ("tp_weaklist", PyObject), - ("tp_del", destructor), + ("tp_methods", Ptr(PyMethodDef)), #S + ("tp_members", Ptr(PyMemberDef)), #S + ("tp_getset", Ptr(PyGetSetDef)), #S + ("tp_base", Ptr(PyTypeObject)), #E + ("tp_dict", PyObject), #U + ("tp_descr_get", descrgetfunc), #N + ("tp_descr_set", descrsetfunc), #N + ("tp_dictoffset", Py_ssize_t), #U + ("tp_init", initproc), #N + ("tp_alloc", allocfunc), #N + ("tp_new", newfunc), #S + ("tp_free", freefunc), #E Low-level free-memory routine + ("tp_is_gc", inquiry), #U For PyObject_IS_GC + ("tp_bases", PyObject),#E + ("tp_mro", PyObject), #U method resolution order + ("tp_cache", PyObject),#S + ("tp_subclasses", PyObject), #U + ("tp_weaklist", PyObject), #U + ("tp_del", destructor), #N ]) cpython_struct("PyTypeObject", PyTypeObjectFields, PyTypeObject) From iammisc at codespeak.net Sat Apr 3 05:21:41 2010 From: iammisc at codespeak.net (iammisc at codespeak.net) Date: Sat, 3 Apr 2010 05:21:41 +0200 (CEST) Subject: [pypy-svn] r73327 - pypy/branch/jit-stackless Message-ID: <20100403032141.B8D36282B90@codespeak.net> Author: iammisc Date: Sat Apr 3 05:21:40 2010 New Revision: 73327 Added: pypy/branch/jit-stackless/ (props changed) - copied from r73326, pypy/trunk/ Log: Creating the jit-stackless branch, which will hold all work relating to integrating PyPy's stackless features with the Just-in-Time compiler. From afa at codespeak.net Sat Apr 3 14:08:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Apr 2010 14:08:52 +0200 (CEST) Subject: [pypy-svn] r73328 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100403120852.E9AC9282B90@codespeak.net> Author: afa Date: Sat Apr 3 14:08:49 2010 New Revision: 73328 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: msvc compiler want declaration at the start of a block Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Sat Apr 3 14:08:49 2010 @@ -119,8 +119,9 @@ PyObject* helper(char* fmt, ...) { va_list va; + PyObject* res; va_start(va, fmt); - PyObject* res = PyString_FromFormatV(fmt, va); + res = PyString_FromFormatV(fmt, va); va_end(va); return res; } From afa at codespeak.net Sat Apr 3 14:10:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Apr 2010 14:10:45 +0200 (CEST) Subject: [pypy-svn] r73329 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403121045.0D6FA282B90@codespeak.net> Author: afa Date: Sat Apr 3 14:10:43 2010 New Revision: 73329 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Easy fixes for test_typeobject, xoraxax probably forgot to submit his code. But I get a segfault... Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 3 14:10:43 2010 @@ -48,7 +48,7 @@ constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING -METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS +METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS METH_VARARGS METH_KEYWORDS Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS """.split() for name in constant_names: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sat Apr 3 14:10:43 2010 @@ -15,14 +15,15 @@ from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ - Py_TPFLAGS_HAVE_CLASS + Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.modsupport import convert_method_defs +from pypy.module.cpyext.modsupport import convert_method_defs, PyCFunction from pypy.module.cpyext.state import State -from pypy.module.cpyext.methodobject import PyDescr_NewWrapper +from pypy.module.cpyext.methodobject import PyDescr_NewWrapper, \ + PyCFunction_NewEx from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr, PyTypeObject, \ @@ -162,11 +163,12 @@ @specialize.memo() def get_new_method_def(space): from pypy.module.cpyext.modsupport import PyMethodDef - ptr = llmemory.malloc(Ptr(PyMethodDef), flavor="raw") + ptr = lltype.malloc(PyMethodDef, flavor="raw") ptr.c_ml_name = rffi.str2charp("__new__") - ptr.c_ml_meth = llhelper(tp_new_wrapper.api_func.functype, - tp_new_wrapper.get_wrapper(space)) - ptr.c_ml_flags = METH_VARARGS | METH_KEYWORDS + ptr.c_ml_meth = rffi.cast(PyCFunction, + llhelper(tp_new_wrapper.api_func.functype, + tp_new_wrapper.api_func.get_wrapper(space))) + rffi.setintfield(ptr, 'c_ml_flags', METH_VARARGS | METH_KEYWORDS) ptr.c_ml_doc = rffi.str2charp("T.__new__(S, ...) -> a new object with type S, a subtype of T") return ptr From afa at codespeak.net Sat Apr 3 14:50:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 3 Apr 2010 14:50:31 +0200 (CEST) Subject: [pypy-svn] r73330 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100403125031.57004282B90@codespeak.net> Author: afa Date: Sat Apr 3 14:50:29 2010 New Revision: 73330 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Skip the rest of this test for the moment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sat Apr 3 14:50:29 2010 @@ -65,6 +65,8 @@ raises(TypeError, "obj.string_member_inplace = 42") raises(TypeError, "del obj.string_member_inplace") + skip("In progress") + a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): From arigo at codespeak.net Sat Apr 3 15:39:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Apr 2010 15:39:36 +0200 (CEST) Subject: [pypy-svn] r73331 - pypy/branch/asmgcc-64 Message-ID: <20100403133936.4A35B282B90@codespeak.net> Author: arigo Date: Sat Apr 3 15:39:34 2010 New Revision: 73331 Removed: pypy/branch/asmgcc-64/ Log: Kill this branch for now. From arigo at codespeak.net Sat Apr 3 15:40:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Apr 2010 15:40:17 +0200 (CEST) Subject: [pypy-svn] r73332 - pypy/branch/asmgcc-64 Message-ID: <20100403134017.2DA31282B90@codespeak.net> Author: arigo Date: Sat Apr 3 15:40:15 2010 New Revision: 73332 Added: pypy/branch/asmgcc-64/ (props changed) - copied from r73331, pypy/trunk/ Log: A branch in which to try a different approach (but still relying on gcc's asm statement). From xoraxax at codespeak.net Sat Apr 3 16:26:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 16:26:29 +0200 (CEST) Subject: [pypy-svn] r73333 - pypy/branch/cpython-extension/pypy/lib Message-ID: <20100403142629.E9F30282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 16:26:28 2010 New Revision: 73333 Modified: pypy/branch/cpython-extension/pypy/lib/identity_dict.py Log: Added copy method to identity_dict. Modified: pypy/branch/cpython-extension/pypy/lib/identity_dict.py ============================================================================== --- pypy/branch/cpython-extension/pypy/lib/identity_dict.py (original) +++ pypy/branch/cpython-extension/pypy/lib/identity_dict.py Sat Apr 3 16:26:28 2010 @@ -30,6 +30,12 @@ def __contains__(self, arg): return id(arg) in self._dict + def copy(self): + d = type(self)() + d.update(self.items()) + assert len(d) == len(self) + return d + class IdentityDictPyPy(object, DictMixin): __slots__ = ["_dict"] From xoraxax at codespeak.net Sat Apr 3 16:26:41 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 16:26:41 +0200 (CEST) Subject: [pypy-svn] r73334 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403142641.B09B0282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 16:26:40 2010 New Revision: 73334 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Updated TODO list. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Sat Apr 3 16:26:40 2010 @@ -1,11 +1,9 @@ - Complete the PyTypeObject initialization code. (see XXX in the code) - - Fill the PyTypeObject slots with method callers. + - Implement further method callers. - Copy the slots from the base. - - Both tasks are necessary to be able to call slots from C code correctly. - - - test_typeobject leaks something - - - Implement subclassing of types defined in C. + - Those tasks are necessary to be able to call slots from C code correctly. + - Additionally, implement further slot wrappers. This is necessary to call + slots of PTOs defined in C. - Use a WeakKeyDictionary to count how often a PyObject is allocated for a given wrapped object and use this to assess whether optimizations are From xoraxax at codespeak.net Sat Apr 3 16:28:28 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 16:28:28 +0200 (CEST) Subject: [pypy-svn] r73335 - in pypy/branch/cpython-extension/pypy: module/cpyext/test rpython/lltypesystem Message-ID: <20100403142828.C0DB6282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 16:28:27 2010 New Revision: 73335 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py Log: Add way to detect whether there are unfreed raw allocations. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Sat Apr 3 16:28:27 2010 @@ -4,7 +4,7 @@ from pypy.conftest import gettestobjspace from pypy.interpreter.error import OperationError -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform from pypy.module.cpyext import api @@ -26,6 +26,14 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] +def set_difference(id_dict1, id_dict2): + d = id_dict1.copy() + for key in id_dict2.keys(): + try: + del d[key] + except KeyError: + pass + return d class AppTestApi: def setup_class(cls): @@ -141,7 +149,10 @@ self.frozen_refcounts = {} for w_obj, obj in state.py_objects_w2r.iteritems(): self.frozen_refcounts[w_obj] = obj.c_ob_refcnt - state.print_refcounts() + #state.print_refcounts() + self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) + self.frozen_lltallocations = lltype.ALLOCATED.copy() + lltype.TRACK_ALLOCATIONS = True def check_and_print_leaks(self): # check for sane refcnts @@ -179,6 +190,16 @@ for w_obj in lost_objects_w: print >>sys.stderr, "Lost object %r" % (w_obj, ) leaking = True + for llvalue in set(ll2ctypes.ALLOCATED.values()) - self.frozen_ll2callocations: + if getattr(llvalue, "_traceback", None): # this means that the allocation should be tracked + leaking = True + print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) + print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) + for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): + leaking = True + print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) + print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) + return leaking Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py Sat Apr 3 16:28:27 2010 @@ -1,3 +1,7 @@ +import StringIO +import traceback +import sys + import py from pypy.rlib.rarithmetic import r_int, r_uint, intmask, r_singlefloat from pypy.rlib.rarithmetic import r_ulonglong, r_longlong, base_int @@ -14,6 +18,7 @@ log = py.log.Producer('lltype') TLS = tlsobject() +TRACK_ALLOCATIONS = False class _uninitialized(object): def __init__(self, TYPE): @@ -1315,27 +1320,49 @@ def _was_freed(self): return False +ALLOCATED = identity_dict() + class _parentable(_container): _kind = "?" __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', '_wrparent', - '__weakref__', - '_storage') + '__weakref__', '_traceback', + '__storage') - def __init__(self, TYPE): + def __init__(self, TYPE, track_allocation=None): self._wrparent = None self._TYPE = TYPE self._storage = True # means "use default storage", as opposed to: # None - container was freed # - using ctypes # (see ll2ctypes.py) + if track_allocation is not False and TRACK_ALLOCATIONS: + self._traceback = self._get_traceback() + ALLOCATED[self] = None + else: + self._traceback = None + + def _get_traceback(self): + frame = sys._getframe().f_back.f_back.f_back.f_back + sio = StringIO.StringIO() + traceback.print_stack(frame, file=sio) + return sio.getvalue() def _free(self): self._check() # no double-frees self._storage = None + def _storage_get(self): + return self.__storage + + def _storage_set(self, value): + self.__storage = value + if value is not True and self in ALLOCATED: + del ALLOCATED[self] + _storage = property(_storage_get, _storage_set) + def _was_freed(self): if self._storage is None: return True @@ -1414,12 +1441,12 @@ __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): - _parentable.__init__(self, TYPE) + def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): + _parentable.__init__(self, TYPE, track_allocation) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) if n is None and TYPE._arrayfld is not None: @@ -1488,12 +1515,12 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None, track_allocation=None): if not isinstance(n, int): raise TypeError, "array length must be an int" if n < 0: raise ValueError, "negative array length" - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation) try: myrange = range(n) except OverflowError: @@ -1790,9 +1817,9 @@ else: initialization = 'malloc' if isinstance(T, Struct): - o = _struct(T, n, initialization=initialization) + o = _struct(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, Array): - o = _array(T, n, initialization=initialization) + o = _array(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, OpaqueType): assert n is None o = _opaque(T, initialization=initialization) From xoraxax at codespeak.net Sat Apr 3 16:52:56 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 16:52:56 +0200 (CEST) Subject: [pypy-svn] r73336 - in pypy/branch/cpython-extension/pypy: module/cpyext/test rpython/lltypesystem Message-ID: <20100403145256.85025282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 16:52:55 2010 New Revision: 73336 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py Log: This test was passing on Py 2.6, but on 2.5 it hit a bug in ctypes. Make a workaround in PyPy. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sat Apr 3 16:52:55 2010 @@ -65,13 +65,17 @@ raises(TypeError, "obj.string_member_inplace = 42") raises(TypeError, "del obj.string_member_inplace") - skip("In progress") + #skip("In progress") # not at all, how does this fail for you amaury? a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): - pass + def baz(self): + return self assert "cannot create" in raises(TypeError, "bar()").value.message fuu = module.FuuType - fuu_inst = fuu(u"abc") + class fuu2(fuu): + def baz(self): + return self + assert fuu2(u"abc").baz().escape() Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py Sat Apr 3 16:52:55 2010 @@ -666,7 +666,8 @@ if T is lltype.Void: return None if isinstance(T, lltype.Ptr): - if not cobj: # NULL pointer + if not cobj or not ctypes.cast(cobj, ctypes.c_void_p).value: # NULL pointer + # CFunctionType.__nonzero__ is broken before Python 2.6 return lltype.nullptr(T.TO) if isinstance(T.TO, lltype.Struct): REAL_TYPE = T.TO From benjamin at codespeak.net Sat Apr 3 19:43:15 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Apr 2010 19:43:15 +0200 (CEST) Subject: [pypy-svn] r73337 - pypy/trunk/pypy/rlib Message-ID: <20100403174315.61C3E282B90@codespeak.net> Author: benjamin Date: Sat Apr 3 19:43:12 2010 New Revision: 73337 Modified: pypy/trunk/pypy/rlib/rcoroutine.py Log: typo Modified: pypy/trunk/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/trunk/pypy/rlib/rcoroutine.py (original) +++ pypy/trunk/pypy/rlib/rcoroutine.py Sat Apr 3 19:43:12 2010 @@ -111,7 +111,7 @@ self.temp_exc = exc def check_for_zombie(self, obj): - return co in self.to_delete + return obj in self.to_delete def postpone_deletion(self, obj): self.to_delete.append(obj) From iammisc at codespeak.net Sat Apr 3 21:14:41 2010 From: iammisc at codespeak.net (iammisc at codespeak.net) Date: Sat, 3 Apr 2010 21:14:41 +0200 (CEST) Subject: [pypy-svn] r73338 - in pypy/branch/jit-stackless/pypy: jit/backend jit/backend/llgraph jit/backend/llsupport jit/backend/x86 jit/metainterp jit/metainterp/test translator/stackless Message-ID: <20100403191441.9A6FE282B90@codespeak.net> Author: iammisc Date: Sat Apr 3 21:14:36 2010 New Revision: 73338 Added: pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py Modified: pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py pypy/branch/jit-stackless/pypy/jit/backend/llgraph/runner.py pypy/branch/jit-stackless/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jit-stackless/pypy/jit/backend/model.py pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py pypy/branch/jit-stackless/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-stackless/pypy/translator/stackless/code.py pypy/branch/jit-stackless/pypy/translator/stackless/frame.py Log: Starting support for jit and stackless integration Modified: pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py Sat Apr 3 21:14:36 2010 @@ -13,7 +13,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr from pypy.rpython.ootypesystem import ootype from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException +from pypy.rpython.llinterp import LLException, type_name from pypy.rpython.extregistry import ExtRegistryEntry from pypy.jit.metainterp import resoperation, executor @@ -22,6 +22,7 @@ from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.rarithmetic import ovfcheck +from pypy.translator.stackless.code import UnwindException import py from pypy.tool.ansi_print import ansi_log @@ -611,6 +612,9 @@ raise GuardFailed def op_guard_no_exception(self, _): + if self.cpu.stackless: + if isinstance(_last_exception, LLException) and type_name(_last_exception.args[0]) == "UnwindException": + raise UnwindException if _last_exception: raise GuardFailed @@ -638,6 +642,9 @@ def op_guard_exception(self, _, expected_exception): global _last_exception + if self.cpu.stackless: + if isinstance(_last_exception, LLException) and type_name(_last_exception.args[0]) == "UnwindException": + raise UnwindException if not self._check_exception(expected_exception): raise GuardFailed res = _last_exception[1] @@ -1131,6 +1138,13 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, _get_error(ZeroDivisionError).args[1]) +def get_unwind_exception(): + return llmemory.cast_ptr_to_adr(_get_error(UnwindException).args[0]) + +def get_unwind_exception_value(): + return lltype.cast_opaque_ptr(llmemory.GCREF, + _get_error(UnwindException).args[1]) + def force(opaque_frame): frame = _from_opaque(opaque_frame) assert not frame._forced @@ -1520,6 +1534,8 @@ setannotation(get_overflow_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(get_zero_division_error, annmodel.SomeAddress()) setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) +setannotation(get_unwind_exception, annmodel.SomeAddress()) +setannotation(get_unwind_exception_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(force, annmodel.SomeInteger()) setannotation(get_forced_token_frame, s_Frame) setannotation(get_frame_forced_token, annmodel.SomeAddress()) Modified: pypy/branch/jit-stackless/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/llgraph/runner.py Sat Apr 3 21:14:36 2010 @@ -79,8 +79,8 @@ translate_support_code=False, annmixlevel=None, gcdescr=None): assert type(opts) is not bool - model.AbstractCPU.__init__(self) self.rtyper = rtyper + model.AbstractCPU.__init__(self) self.translate_support_code = translate_support_code self.stats = stats or MiniStats() self.stats.exec_counters = {} @@ -287,6 +287,10 @@ return (self.cast_adr_to_int(llimpl.get_zero_division_error()), llimpl.get_zero_division_error_value()) + def get_unwind_exception(self): + return (self.cast_adr_to_int(llimpl.get_unwind_exception()), + llimpl.get_unwind_exception_value()) + def sizeof(self, S): assert not isinstance(S, lltype.Ptr) return self.getdescr(symbolic.get_size(S)) @@ -586,6 +590,11 @@ return (ootype.cast_to_object(ll_err.args[0]), ootype.cast_to_object(ll_err.args[1])) + def get_unwind_exception(self): + ll_err = llimpl._get_error(UnwindException) + return (ootype.cast_to_object(ll_err.args[0]), + ootype.cast_to_object(ll_err.args[1])) + def do_new_with_vtable(self, clsbox): cls = clsbox.getref_base() typedescr = self.class_sizes[cls] Modified: pypy/branch/jit-stackless/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/llsupport/llmodel.py Sat Apr 3 21:14:36 2010 @@ -14,6 +14,7 @@ from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr from pypy.jit.backend.llsupport.descr import get_call_descr, BaseCallDescr from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.translator.stackless.code import UnwindException empty_int_box = BoxInt(0) @@ -24,10 +25,11 @@ gcdescr=None): assert type(opts) is not bool self.opts = opts + self.rtyper = rtyper from pypy.jit.backend.llsupport.gc import get_ll_description AbstractCPU.__init__(self) - self.rtyper = rtyper + self.stats = stats self.translate_support_code = translate_support_code if translate_support_code: @@ -43,6 +45,7 @@ translate_support_code) self._setup_prebuilt_error('ovf', OverflowError) self._setup_prebuilt_error('zer', ZeroDivisionError) + self._setup_prebuilt_error('stku', UnwindException) if translate_support_code: self._setup_exception_handling_translated() else: @@ -247,6 +250,12 @@ self._zer_error_inst) return zer_vtable, zer_inst + def get_unwind_exception(self): + stku_vtable = self.cast_adr_to_int(self._stku_error_vtable) + stku_inst = lltype.cast_opaque_ptr(llmemory.GCREF, + zelf._stku_error_inst) + return stku_vtable, stku_inst + # ____________________________________________________________ def do_arraylen_gc(self, arraybox, arraydescr): Modified: pypy/branch/jit-stackless/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/model.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/model.py Sat Apr 3 21:14:36 2010 @@ -8,9 +8,15 @@ portal_calldescr = None done_with_this_frame_int_v = -1 + stackless = False # this is set to true in __init__, if we are indeed a stackless build + def __init__(self): self.fail_descr_list = [] + if self.rtyper is not None: + config = self.rtyper.annotator.translator.config + self.stackless = True + def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) n = descr.index Modified: pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py Sat Apr 3 21:14:36 2010 @@ -17,6 +17,8 @@ from pypy.jit.backend.x86.support import values_array from pypy.rlib.debug import debug_print from pypy.rlib import rgc +from pypy.translator.stackless.frame import STATE_HEADER +from pypy.translator.stackless.code import global_state # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -30,6 +32,12 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) +JIT_STATE_HEADER = lltype.Struct('JIT_STATE_HEADER', + ('header', STATE_HEADER), + ('saved_ints', lltype.Ptr(lltype.GcArray(lltype.Signed))), + ('saved_floats', lltype.Ptr(lltype.GcArray(lltype.Float))), + ('saved_refs', lltype.Ptr(lltype.GcArray(lltype.Ptr)))) + class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 @@ -121,6 +129,7 @@ self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 self.setup_failure_recovery() + self.setup_unwinder_recovery() def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -159,13 +168,19 @@ # 'mc2' is for guard recovery code self.mc = MachineCodeBlockWrapper(self.mc_size, self.cpu.profile_agent) self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self._build_failure_recovery(False) self._build_failure_recovery(True) + self._build_unwinder_recovery(False) + self._build_unwinder_recovery(True) if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) + self._build_unwinder_recovery(False, withfloats=True) + self._build_unwinder_recovery(True, withfloats=True) codebuf.ensure_sse2_floats() self._build_float_constants() + if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() @@ -490,7 +505,11 @@ dispatch_opnum = guard_opnum else: dispatch_opnum = op.opnum - res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + if dispatch_opnum == "GUARD_EXCEPTION" or dispatch_opnum == "GUARD_NO_EXCEPTION": + res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, + arglocs, resloc, faillocs) + else: + res = genop_guard_list[dispatch_opnum](self, op, guard_op, failaddr, arglocs, resloc) faildescr._x86_adr_jump_offset = res @@ -952,14 +971,25 @@ genop_guard_guard_nonnull = genop_guard_guard_true def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, - locs, ign_2): + locs, ign_2, faillocs): + loc = locs[0] + if self.cpu.stackless: + unwinder_addr = self.generate_unwinder_failure(guard_op.descr, guard_op.fail_args, faillocs) + stku_type, stku_instance = self.cpu.get_unwind_exception() + self.mc.CMP(heap(self.cpu.pos_exception()), imm(stku_type)) + self.mc.JE(rel32(unwinder_addr)) # if this is indeed an unwind exception jump to saver code self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) def genop_guard_guard_exception(self, ign_1, guard_op, addr, - locs, resloc): + locs, resloc, faillocs): loc = locs[0] loc1 = locs[1] + if self.cpu.stackless: + unwinder_addr = self.generate_unwinder_failure(guard_op.descr, guard_op.fail_args, faillocs) + stku_type, stku_instance = self.cpu.get_unwind_exception() + self.mc.CMP(heap(self.cpu.pos_exception()), imm(stku_type)) + self.mc.JE(rel32(unwinder_addr)) # if this is indeed an unwind exception jump to saver code self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) addr = self.implement_guard(addr, self.mc.JNE) @@ -1043,6 +1073,12 @@ return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + self.generate_quick_failure_generic(self.failure_recovery_code, faildescr, failargs, fail_locs, exc) + + def generate_unwinder_failure(self, faildescr, failargs, fail_locs): + self.generate_quick_failure_generic(self.stackless_recovery_func, faildescr, failargs, fail_locs, True) + + def generate_quick_failure_generic(self, recovery_func, faildescr, failargs, fail_locs, exc): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -1060,7 +1096,7 @@ if box is not None and box.type == FLOAT: withfloats = True break - mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + mc.CALL(rel32(recovery_func[exc + 2 * withfloats])) # write tight data that describes the failure recovery faildescr._x86_failure_recovery_bytecode = mc.tell() self.write_failure_recovery_description(mc, failargs, fail_locs) @@ -1191,6 +1227,40 @@ return boxes @rgc.no_collect + def unwind_frame_values(self, bytecode, frame_addr, allregisters): + # Okay allocate a frame for us, + + # then set the proper values in global_state + + bytecode = rffi.cast(rffi.UCHARP, bytecode) + boxes = [] + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + kind = code & 3 + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + index = len(boxes) + # Now kind contains information at the type of data that will go here + if kind != self.DESCR_SPECIAL: + kinds.append(kind) + else: + if code == self.CODE_STOP: + break + assert code == self.CODE_HOLE, "Bad code" + + # Now kinds contains the types we need to store + state_header = lltype.malloc(JIT_STATE_HEADER, zero=True) + + state_header.saved_ints = lltype.malloc(lltype.GcArray(lltype.Signed), len([x for x in kinds if x == self.DESCR_INT])) + state_header.saved_floats = lltype.malloc(lltype.GcArray(lltype.Float), len([x for x in kinds if x == self.DESCR_FLOAT])) + state_header.saved_floats = lltype.malloc(lltype.GcArray(lltype.Ptr), len([x for x in kinds if x == self.DESCR_REF])) + + + + @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): # no malloc allowed here!! self.fail_ebp = allregisters[16 + ebp.op] @@ -1269,14 +1339,40 @@ self.failure_recovery_func = failure_recovery_func self.failure_recovery_code = [0, 0, 0, 0] + def setup_unwinder_recovery(self): + + @rgc.no_collect + def unwinder_recovery_func(registers): + stack_at_ebp = registers[ebp.op] + bytecode = rffi.cast(rffi.UCHARP, registers[8]) + allregisters = rffi.ptradd(registers, -16) + return self.unwind_frame_values(bytecode, stack_at_ebp, allregisters) + + self.unwinder_recovery_func = unwinder_recovery_func + self.unwinder_recovery_code = [0, 0, 0, 0] + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], lltype.Signed)) + _UNWINDER_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], + lltype.Signed)) def _build_failure_recovery(self, exc, withfloats=False): failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) + recovery_addr = self._build_failure_recovery_generic(failure_recovery_func, exc, withfloats=withfloats) + self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + + def _build_unwinder_recovery(self, exc, withfloats=False): + unwinder_recovery_func = llhelper(self._UNWINDER_RECOVERY_FUNC, + self.unwinder_recovery_func) + unwinder_recovery_func = rffi.cast(lltype.Signed, + unwinder_recovery_func) + recovery_addr = self._build_failure_recovery_generic(unwinder_recovery_func, exc, withfloats=withfloats) + self.unwinder_recovery_code[exc + 2 * withfloats] = recovery_addr + + def _build_failure_recovery_generic(self, recovery_func, exc, withfloats=False): mc = self.mc2._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). @@ -1312,7 +1408,7 @@ # generate_quick_failure(). XXX misaligned stack in the call, but # it's ok because failure_recovery_func is not calling anything more mc.PUSH(esi) - mc.CALL(rel32(failure_recovery_func)) + mc.CALL(rel32(recovery_func)) # returns in eax the fail_index # now we return from the complete frame, which starts from @@ -1325,7 +1421,7 @@ mc.POP(ebp) # [ebp] mc.RET() self.mc2.done() - self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + return recovery_addr def generate_failure(self, fail_index, locs, exc, locs_are_ref): mc = self.mc Modified: pypy/branch/jit-stackless/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-stackless/pypy/jit/metainterp/pyjitpl.py Sat Apr 3 21:14:36 2010 @@ -17,6 +17,7 @@ from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp +from pypy.translator.stackless.code import UnwindException # ____________________________________________________________ @@ -1404,6 +1405,12 @@ self.cpu.ts.get_exception_box(etype), self.cpu.ts.get_exc_value_box(evalue)) + def raise_unwind_exception(self): + etype, evalue = self.cpu.get_unwind_exception() + return self.finishframe_exception( + self.cpu.ts.get_exception_box(etype), + self.cpu.ts.get_exc_value_box(evalue)) + def create_empty_history(self): warmrunnerstate = self.staticdata.state self.history = history.History() @@ -1888,13 +1895,19 @@ self.cpu.clear_exception() frame = self.framestack[-1] if etype: - exception_box = self.cpu.ts.get_exception_box(etype) - exc_value_box = self.cpu.ts.get_exc_value_box(evalue) - op = frame.generate_guard(frame.pc, rop.GUARD_EXCEPTION, - None, [exception_box]) - if op: - op.result = exc_value_box - return self.finishframe_exception(exception_box, exc_value_box) + # Handle UnwindException here, because we need to invoke + # special state saver code in order to handle stackless. + if etype is UnwindException: + assert self.cpu.stackless + return self.finishframe_unwind() + else: + exception_box = self.cpu.ts.get_exception_box(etype) + exc_value_box = self.cpu.ts.get_exc_value_box(evalue) + op = frame.generate_guard(frame.pc, rop.GUARD_EXCEPTION, + None, [exception_box]) + if op: + op.result = exc_value_box + return self.finishframe_exception(exception_box, exc_value_box) else: frame.generate_guard(frame.pc, rop.GUARD_NO_EXCEPTION, None, []) return False Added: pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py ============================================================================== --- (empty file) +++ pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py Sat Apr 3 21:14:36 2010 @@ -0,0 +1,59 @@ +import py +import os +import sys +from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE, OPTIMIZER_FULL +from pypy.rlib.objectmodel import compute_hash +from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats, jittify_and_run, WarmRunnerDesc +from pypy.rpython.lltypesystem import lltype +from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin +from pypy.jit.metainterp.policy import StopAtXPolicy +from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.llgraph import runner +from pypy.jit.metainterp import history +from pypy.translator.stackless.code import yield_current_frame_to_caller +from pypy.translator.translator import TranslationContext, graphof +from pypy.annotation.listdef import s_list_of_strings +from pypy.translator.stackless.transform import StacklessTransformer, FrameTyper +from pypy.translator.c.genc import CStandaloneBuilder +from pypy.translator.c import gc +from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache +from pypy import conftest +from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE +from pypy.translator.stackless.code import UnwindException + +optimizer = OPTIMIZER_SIMPLE + +def meta_interp(f, args=(), policy=None): + return ll_meta_interp(f, args, optimizer=optimizer, + policy=policy, + CPUClass=runner.LLtypeCPU, + type_system='lltype') + +def test_simple_unwind(): + myjitdriver = JitDriver(greens = [], reds = ['y', 'x']) + + def g(z): + if z % 2: + parent = yield_current_frame_to_caller() + return parent + else: + return None + + def f(y): + x = 0 + while y > 0: + myjitdriver.can_enter_jit(x=x, y=y) + myjitdriver.jit_merge_point(x=x, y=y) + + try: + res = g(y) + if res is not None: + res = res.switch() + if res is None: + x += 1 + except UnwindException: + pass + y -= 1 + return x + res = meta_interp(f, [100]) + assert res == 100 Modified: pypy/branch/jit-stackless/pypy/translator/stackless/code.py ============================================================================== --- pypy/branch/jit-stackless/pypy/translator/stackless/code.py (original) +++ pypy/branch/jit-stackless/pypy/translator/stackless/code.py Sat Apr 3 21:14:36 2010 @@ -391,10 +391,18 @@ while True: prevdepth = pending.f_depth - 1 back = pending.f_back - decoded = frame.decodestate(pending.f_restart) - (fn, global_state.restart_substate, signature_index) = decoded + enter_jit = False + if pending.f_restart_addr != llmemory.NULL: + # we're going to enter directly into jitted code + enter_jit = True + else: + decoded = frame.decodestate(pending.f_restart) + (fn, global_state.restart_substate, signature_index) = decoded try: - call_function(fn, signature_index) + if enter_jit: + pass + else: # enter native code + call_function(fn, signature_index) except UnwindException, u: #XXX annotation support needed u.frame_bottom.f_back = back depth = prevdepth + u.depth Modified: pypy/branch/jit-stackless/pypy/translator/stackless/frame.py ============================================================================== --- pypy/branch/jit-stackless/pypy/translator/stackless/frame.py (original) +++ pypy/branch/jit-stackless/pypy/translator/stackless/frame.py Sat Apr 3 21:14:36 2010 @@ -65,6 +65,7 @@ STATE_HEADER = lltype.GcStruct('state_header', ('f_back', lltype.Ptr(lltype.GcForwardReference())), ('f_restart', lltype.Signed), + ('f_restart_addr', lltype.Signed), ('f_depth', lltype.Signed)) STATE_HEADER.f_back.TO.become(STATE_HEADER) From benjamin at codespeak.net Sat Apr 3 21:33:14 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Apr 2010 21:33:14 +0200 (CEST) Subject: [pypy-svn] r73339 - pypy/branch/improve-jit-interp Message-ID: <20100403193314.65670282B90@codespeak.net> Author: benjamin Date: Sat Apr 3 21:33:12 2010 New Revision: 73339 Added: pypy/branch/improve-jit-interp/ (props changed) - copied from r73338, pypy/trunk/ Log: a branch to try and improve the metainterp From xoraxax at codespeak.net Sat Apr 3 22:36:08 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 22:36:08 +0200 (CEST) Subject: [pypy-svn] r73340 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100403203608.1C787282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 22:36:06 2010 New Revision: 73340 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Log: Added new PyErr functions and PyMem functions, working towards _sre compliance. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sat Apr 3 22:36:06 2010 @@ -65,6 +65,7 @@ #include "intobject.h" #include "unicodeobject.h" #include "eval.h" +#include "pymem.h" // XXX This shouldn't be included here #include "structmember.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Sat Apr 3 22:36:06 2010 @@ -0,0 +1,12 @@ +#include + + +#define PyMem_MALLOC(n) malloc((n) ? (n) : 1) +#define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) +#define PyMem_FREE free + +/* XXX use obmalloc like cpython does */ +#define PyObject_MALLOC PyMem_MALLOC +#define PyObject_REALLOC PyMem_REALLOC +#define PyObject_FREE PyMem_FREE + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Sat Apr 3 22:36:06 2010 @@ -56,6 +56,34 @@ def PyErr_BadInternalCall(space): raise OperationError(space.w_SystemError, space.wrap("Bad internal call!")) + at cpython_api([], rffi.INT_real, error=-1) +def PyErr_CheckSignals(space): + """ + This function interacts with Python's signal handling. It checks whether a + signal has been sent to the processes and if so, invokes the corresponding + signal handler. If the signal module is supported, this can invoke a + signal handler written in Python. In all cases, the default effect for + SIGINT is to raise the KeyboardInterrupt exception. If an + exception is raised the error indicator is set and the function returns -1; + otherwise the function returns 0. The error indicator may or may not be + cleared if it was previously set.""" + # XXX implement me + return 0 + + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyErr_NoMemory(space): + """This is a shorthand for PyErr_SetNone(PyExc_MemoryError); it returns NULL + so an object allocation function can write return PyErr_NoMemory(); when it + runs out of memory. + Return value: always NULL.""" + PyErr_SetNone(space.w_MemoryError) + + at cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL) +def PyErr_SetNone(space, w_type): + """This is a shorthand for PyErr_SetObject(type, Py_None).""" + PyErr_SetObject(w_type, space.w_None) + + @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyErr_GivenExceptionMatches(space, w_given, w_exc): """Return true if the given exception matches the exception in exc. If From xoraxax at codespeak.net Sat Apr 3 22:43:14 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 3 Apr 2010 22:43:14 +0200 (CEST) Subject: [pypy-svn] r73341 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100403204314.93B8D282B90@codespeak.net> Author: xoraxax Date: Sat Apr 3 22:43:12 2010 New Revision: 73341 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Add new types to stubgen and run stubgen.py. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubgen.py Sat Apr 3 22:43:12 2010 @@ -26,6 +26,8 @@ "Py_ssize_t*": "Py_ssize_t", "...": "...", "char": "lltype.Char", + "long": "lltype.Signed", + "": "", } C_TYPE_TO_PYPY_TYPE_ARGS = C_TYPE_TO_PYPY_TYPE.copy() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sat Apr 3 22:43:12 2010 @@ -1328,13 +1328,6 @@ def PyWrapper_New(space, , ): raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyDict_CheckExact(space, p): - """Return true if p is a dict object, but not an instance of a subtype of - the dict type. - """ - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyDictProxy_New(space, dict): """Return a proxy object for a mapping which enforces read-only behavior. @@ -1362,13 +1355,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real) -def PyDict_SetItem(space, p, key, val): - """Insert value into the dictionary p with a key of key. key must be - hashable; if it isn't, TypeError will be raised. Return - 0 on success or -1 on failure.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], rffi.INT_real) def PyDict_DelItem(space, p, key): """Remove the entry in dictionary p with key key. key must be hashable; @@ -1382,12 +1368,6 @@ key. Return 0 on success or -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject, borrowed=True) -def PyDict_GetItem(space, p, key): - """Return the object from dictionary p which has a key key. Return NULL - if the key key is not present, but without setting an exception.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyDict_Items(space, p): """Return a PyListObject containing all the items from the @@ -1419,7 +1399,7 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, {PyObject**}, {PyObject**}], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real) def PyDict_Next(space, p, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 @@ -1511,7 +1491,7 @@ type, value and traceback of the printed exception, respectively.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyErr_Print(space, ): """Alias for PyErr_PrintEx(1).""" raise NotImplementedError @@ -1525,7 +1505,7 @@ The delayed normalization is implemented to improve performance.""" raise NotImplementedError - at cpython_api([{PyObject**}, {PyObject**}, {PyObject**}], lltype.Void) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) def PyErr_Fetch(space, ptype, pvalue, ptraceback): """Retrieve the error indicator into three variables whose addresses are passed. If the error indicator is not set, set all three variables to NULL. If it is @@ -1701,26 +1681,13 @@ Return value: always NULL.""" raise NotImplementedError - at cpython_api([PyObject], lltype.Void) -def PyErr_SetNone(space, type): - """This is a shorthand for PyErr_SetObject(type, Py_None).""" - raise NotImplementedError - - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyErr_BadArgument(space, ): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where message indicates that a built-in operation was invoked with an illegal argument. It is mostly for internal use.""" raise NotImplementedError - at cpython_api([{}], PyObject) -def PyErr_NoMemory(space, ): - """This is a shorthand for PyErr_SetNone(PyExc_MemoryError); it returns NULL - so an object allocation function can write return PyErr_NoMemory(); when it - runs out of memory. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyErr_SetFromErrnoWithFilename(space, type, filename): """Similar to PyErr_SetFromErrno(), with the additional behavior that if @@ -1832,23 +1799,7 @@ """ raise NotImplementedError - at cpython_api([{}], rffi.INT_real) -def PyErr_CheckSignals(space, ): - """ - - - - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the signal module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - SIGINT is to raise the KeyboardInterrupt exception. If an - exception is raised the error indicator is set and the function returns -1; - otherwise the function returns 0. The error indicator may or may not be - cleared if it was previously set.""" - raise NotImplementedError - - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyErr_SetInterrupt(space, ): """ @@ -1925,7 +1876,7 @@ limit.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def Py_LeaveRecursiveCall(space, ): """Ends a Py_EnterRecursiveCall(). Must be called once for each successful invocation of Py_EnterRecursiveCall().""" @@ -2114,19 +2065,19 @@ """ raise NotImplementedError - at cpython_api([{}], {double}) + at cpython_api([], {double}) def PyFloat_GetMax(space, ): """Return the maximum representable finite float DBL_MAX as C double. """ raise NotImplementedError - at cpython_api([{}], {double}) + at cpython_api([], {double}) def PyFloat_GetMin(space, ): """Return the minimum normalized positive float DBL_MIN as C double. """ raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyFloat_ClearFreeList(space, ): """Clear the float free list. Return the number of items that could not be freed. @@ -2446,14 +2397,14 @@ name is removed from sys.modules in error cases.""" raise NotImplementedError - at cpython_api([{}], {long}) + at cpython_api([], lltype.Signed) def PyImport_GetMagicNumber(space, ): """Return the magic number for Python bytecode files (a.k.a. .pyc and .pyo files). The magic number should be present in the first four bytes of the bytecode file, in little-endian byte order.""" raise NotImplementedError - at cpython_api([{}], PyObject, borrowed=True) + at cpython_api([], PyObject, borrowed=True) def PyImport_GetModuleDict(space, ): """Return the dictionary used for the module administration (a.k.a. sys.modules). Note that this is a per-interpreter variable.""" @@ -2471,17 +2422,17 @@ """ raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def _PyImport_Init(space, ): """Initialize the import mechanism. For internal use only.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyImport_Cleanup(space, ): """Empty the module table. For internal use only.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def _PyImport_Fini(space, ): """Finalize the import mechanism. For internal use only.""" raise NotImplementedError @@ -2525,7 +2476,7 @@ internal table. This should be called before Py_Initialize().""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def Py_Initialize(space, ): """ @@ -2551,7 +2502,7 @@ """ raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def Py_Finalize(space, ): """Undo all initializations made by Py_Initialize() and subsequent use of Python/C API functions, and destroy all sub-interpreters (see @@ -2581,7 +2532,7 @@ Py_Finalize() more than once.""" raise NotImplementedError - at cpython_api([{}], {PyThreadState*}) + at cpython_api([], {PyThreadState*}) def Py_NewInterpreter(space, ): """ @@ -2679,7 +2630,7 @@ interpreter will change the contents of this storage.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetProgramName(space, ): """ @@ -2690,7 +2641,7 @@ value.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with @@ -2703,7 +2654,7 @@ It is only useful on Unix. See also the next function.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetExecPrefix(space, ): """Return the exec-prefix for installed platform-dependent files. This is derived through a number of complicated rules from the program name set with @@ -2738,7 +2689,7 @@ platform.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetProgramFullPath(space, ): """ @@ -2751,7 +2702,7 @@ to Python code as sys.executable.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetPath(space, ): """ @@ -2770,7 +2721,7 @@ XXX should give the exact rules""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetVersion(space, ): """Return the version of this Python interpreter. This is a string that looks something like @@ -2787,7 +2738,7 @@ modify its value. The value is available to Python code as sys.version.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetPlatform(space, ): """ @@ -2802,7 +2753,7 @@ to Python code as sys.platform.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetCopyright(space, ): """Return the official copyright string for the current Python version, for example @@ -2816,7 +2767,7 @@ value. The value is available to Python code as sys.copyright.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetCompiler(space, ): """Return an indication of the compiler used to build the current Python version, in square brackets, for example: @@ -2832,7 +2783,7 @@ sys.version.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetBuildInfo(space, ): """Return information about the sequence number and build date and time of the current Python interpreter instance, for example @@ -2881,14 +2832,14 @@ this storage.""" raise NotImplementedError - at cpython_api([{}], rffi.CCHARP) + at cpython_api([], rffi.CCHARP) def Py_GetPythonHome(space, ): """Return the default "home", that is, the value set by a previous call to Py_SetPythonHome(), or the value of the PYTHONHOME environment variable if it is set.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyEval_InitThreads(space, ): """ @@ -2928,7 +2879,7 @@ This function is not available when thread support is disabled at compile time.""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyEval_ThreadsInitialized(space, ): """Returns a non-zero value if PyEval_InitThreads() has been called. This function can be called without holding the GIL, and therefore can be used to @@ -2937,14 +2888,14 @@ """ raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyEval_AcquireLock(space, ): """Acquire the global interpreter lock. The lock must have been created earlier. If this thread already has the lock, a deadlock ensues. This function is not available when thread support is disabled at compile time.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyEval_ReleaseLock(space, ): """Release the global interpreter lock. The lock must have been created earlier. This function is not available when thread support is disabled at compile time.""" @@ -2968,7 +2919,7 @@ compile time.""" raise NotImplementedError - at cpython_api([{}], {PyThreadState*}) + at cpython_api([], {PyThreadState*}) def PyEval_SaveThread(space, ): """Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to NULL, returning the @@ -2986,14 +2937,14 @@ when thread support is disabled at compile time.)""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyEval_ReInitThreads(space, ): """This function is called from PyOS_AfterFork() to ensure that newly created child processes don't hold locks referring to threads which are not running in the child process.""" raise NotImplementedError - at cpython_api([{}], {PyInterpreterState*}) + at cpython_api([], {PyInterpreterState*}) def PyInterpreterState_New(space, ): """Create a new interpreter state object. The global interpreter lock need not be held, but may be held if it is necessary to serialize calls to this @@ -3033,7 +2984,7 @@ PyThreadState_Clear().""" raise NotImplementedError - at cpython_api([{}], {PyThreadState*}) + at cpython_api([], {PyThreadState*}) def PyThreadState_Get(space, ): """Return the current thread state. The global interpreter lock must be held. When the current thread state is NULL, this issues a fatal error (so that @@ -3046,7 +2997,7 @@ tstate, which may be NULL. The global interpreter lock must be held.""" raise NotImplementedError - at cpython_api([{}], PyObject, borrowed=True) + at cpython_api([], PyObject, borrowed=True) def PyThreadState_GetDict(space, ): """Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in @@ -3058,7 +3009,7 @@ meant that an exception was raised.""" raise NotImplementedError - at cpython_api([{long}, PyObject], rffi.INT_real) + at cpython_api([lltype.Signed, PyObject], rffi.INT_real) def PyThreadState_SetAsyncExc(space, id, exc): """Asynchronously raise an exception in a thread. The id argument is the thread id of the target thread; exc is the exception object to be raised. This @@ -3070,7 +3021,7 @@ """ raise NotImplementedError - at cpython_api([{}], {PyGILState_STATE}) + at cpython_api([], {PyGILState_STATE}) def PyGILState_Ensure(space, ): """Ensure that the current thread is ready to call the Python C API regardless of the current state of Python, or of the global interpreter lock. This may @@ -3217,7 +3168,7 @@ defined.""" raise NotImplementedError - at cpython_api([{}], {PyInterpreterState*}) + at cpython_api([], {PyInterpreterState*}) def PyInterpreterState_Head(space, ): """Return the interpreter state object at the head of the list of all such objects. """ @@ -3244,13 +3195,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyInt_CheckExact(space, o): - """Return true if o is of type PyInt_Type, but not a subtype of - PyInt_Type. - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, {char**}, rffi.INT_real], PyObject) def PyInt_FromString(space, str, pend, base): """Return a new PyIntObject or PyLongObject based on the string @@ -3268,17 +3212,6 @@ returned in this case.""" raise NotImplementedError - at cpython_api([{long}], PyObject) -def PyInt_FromLong(space, ival): - """Create a new integer object with a value of ival. - - The current implementation keeps an array of integer objects for all integers - between -5 and 256, when you create an int in that range you actually - just get back a reference to the existing object. So it should be possible to - change the value of 1. I suspect the behaviour of Python in this case is - undefined. :-)""" - raise NotImplementedError - @cpython_api([Py_ssize_t], PyObject) def PyInt_FromSsize_t(space, ival): """Create a new integer object with a value of ival. If the value is larger @@ -3294,15 +3227,7 @@ """ raise NotImplementedError - at cpython_api([PyObject], {long}) -def PyInt_AsLong(space, io): - """Will first attempt to cast the object to a PyIntObject, if it is not - already one, and then return its value. If there is an error, -1 is - returned, and the caller should check PyErr_Occurred() to find out whether - there was an error, or whether the value just happened to be -1.""" - raise NotImplementedError - - at cpython_api([PyObject], {long}) + at cpython_api([PyObject], lltype.Signed) def PyInt_AS_LONG(space, io): """Return the value of the object io. No error checking is performed.""" raise NotImplementedError @@ -3331,7 +3256,7 @@ """ raise NotImplementedError - at cpython_api([{}], {long}) + at cpython_api([], lltype.Signed) def PyInt_GetMax(space, ): """ @@ -3341,7 +3266,7 @@ (LONG_MAX, as defined in the system header files).""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyInt_ClearFreeList(space, ): """Clear the integer free list. Return the number of items that could not be freed. @@ -3391,21 +3316,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyList_Check(space, p): - """Return true if p is a list object or an instance of a subtype of the list - type. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyList_CheckExact(space, p): - """Return true if p is a list object, but not an instance of a subtype of - the list type. - """ - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyList_Size(space, list): """ @@ -3446,18 +3356,6 @@ your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real) -def PyList_SetItem(space, list, index, item): - """Set the item at index index in list to item. Return 0 on success - or -1 on failure. - - This function "steals" a reference to item and discards a reference to - an item already in the list at the affected position. - - This function used an int for index. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, PyObject], lltype.Void) def PyList_SET_ITEM(space, list, i, o): """Macro form of PyList_SetItem() without error checking. This is @@ -3482,13 +3380,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) -def PyList_Append(space, list, item): - """Append the object item at the end of list list. Return 0 if - successful; return -1 and set an exception if unsuccessful. Analogous - to list.append(item).""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PyList_GetSlice(space, list, low, high): """Return a list of the objects in list containing the objects between low @@ -3624,7 +3515,7 @@ If the integer is larger than LONG_MAX, a positive long integer is returned.""" raise NotImplementedError - at cpython_api([PyObject], {long}) + at cpython_api([PyObject], lltype.Signed) def PyLong_AsLong(space, pylong): """ @@ -3635,7 +3526,7 @@ and -1 will be returned.""" raise NotImplementedError - at cpython_api([PyObject, {int*}], {long}) + at cpython_api([PyObject, {int*}], lltype.Signed) def PyLong_AsLongAndOverflow(space, pylong, overflow): """Return a C long representation of the contents of pylong. If pylong is greater than LONG_MAX or less @@ -3821,7 +3712,7 @@ This is the equivalent of the Python statement o[key] = v.""" raise NotImplementedError - at cpython_api([{long}, {FILE*}, rffi.INT_real], lltype.Void) + at cpython_api([lltype.Signed, {FILE*}, rffi.INT_real], lltype.Void) def PyMarshal_WriteLongToFile(space, value, file, version): """Marshal a long integer, value, to file. This will only write the least-significant 32 bits of value; regardless of the size of the @@ -3844,7 +3735,7 @@ version indicates the file format.""" raise NotImplementedError - at cpython_api([{FILE*}], {long}) + at cpython_api([{FILE*}], lltype.Signed) def PyMarshal_ReadLongFromFile(space, file): """Return a C long from the data stream in a FILE* opened for reading. Only a 32-bit value can be read in using this function, @@ -3984,7 +3875,7 @@ """Macro version of PyMethod_Self() which avoids error checking.""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyMethod_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ @@ -4027,7 +3918,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, {long}], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, lltype.Signed], rffi.INT_real) def PyModule_AddIntConstant(space, module, name, value): """Add an integer constant to module as name. This convenience function can be used from the module's initialization function. Return -1 on error, 0 on @@ -4292,7 +4183,7 @@ the Python statement o1 |= o2.""" raise NotImplementedError - at cpython_api([{PyObject**}, {PyObject**}], rffi.INT_real) + at cpython_api([PyObjectP, PyObjectP], rffi.INT_real) def PyNumber_Coerce(space, p1, p2): """ @@ -4308,7 +4199,7 @@ &o2) is equivalent to the Python statement o1, o2 = coerce(o1, o2).""" raise NotImplementedError - at cpython_api([{PyObject**}, {PyObject**}], rffi.INT_real) + at cpython_api([PyObjectP, PyObjectP], rffi.INT_real) def PyNumber_CoerceEx(space, p1, p2): """This function is similar to PyNumber_Coerce(), except that it returns 1 when the conversion is not possible and when no error is raised. @@ -4337,16 +4228,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyNumber_Float(space, o): - """ - - - - Returns the o converted to a float object on success, or NULL on failure. - This is the equivalent of the Python expression float(o).""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyNumber_Index(space, o): """Returns the o converted to a Python int or long on success or NULL with a TypeError exception raised on failure. @@ -4701,7 +4582,7 @@ """ raise NotImplementedError - at cpython_api([PyObject], {long}) + at cpython_api([PyObject], lltype.Signed) def PyObject_Hash(space, o): """ @@ -4711,7 +4592,7 @@ This is the equivalent of the Python expression hash(o).""" raise NotImplementedError - at cpython_api([PyObject], {long}) + at cpython_api([PyObject], lltype.Signed) def PyObject_HashNotImplemented(space, o): """Set a TypeError indicating that type(o) is not hashable and return -1. This function receives special treatment when stored in a tp_hash slot, @@ -4800,12 +4681,42 @@ raise NotImplementedError @cpython_api([PyObject], lltype.Void) +def Py_INCREF(space, o): + """Increment the reference count for object o. The object must not be NULL; if + you aren't sure that it isn't NULL, use Py_XINCREF().""" + raise NotImplementedError + + at cpython_api([PyObject], lltype.Void) def Py_XINCREF(space, o): """Increment the reference count for object o. The object may be NULL, in which case the macro has no effect.""" raise NotImplementedError @cpython_api([PyObject], lltype.Void) +def Py_DECREF(space, o): + """Decrement the reference count for object o. The object must not be NULL; if + you aren't sure that it isn't NULL, use Py_XDECREF(). If the reference + count reaches zero, the object's type's deallocation function (which must not be + NULL) is invoked. + + The deallocation function can cause arbitrary Python code to be invoked (e.g. + when a class instance with a __del__() method is deallocated). While + exceptions in such code are not propagated, the executed code has free access to + all Python global variables. This means that any object that is reachable from + a global variable should be in a consistent state before Py_DECREF() is + invoked. For example, code to delete an object from a list should copy a + reference to the deleted object in a temporary variable, update the list data + structure, and then call Py_DECREF() for the temporary variable.""" + raise NotImplementedError + + at cpython_api([PyObject], lltype.Void) +def Py_XDECREF(space, o): + """Decrement the reference count for object o. The object may be NULL, in + which case the macro has no effect; otherwise the effect is the same as for + Py_DECREF(), and the same warning applies.""" + raise NotImplementedError + + at cpython_api([PyObject], lltype.Void) def Py_CLEAR(space, o): """Decrement the reference count for object o. The object may be NULL, in which case the macro has no effect; otherwise the effect is the same as for @@ -4819,25 +4730,25 @@ """ raise NotImplementedError - at cpython_api([{}], PyObject, borrowed=True) + at cpython_api([], PyObject, borrowed=True) def PyEval_GetBuiltins(space, ): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing.""" raise NotImplementedError - at cpython_api([{}], PyObject, borrowed=True) + at cpython_api([], PyObject, borrowed=True) def PyEval_GetLocals(space, ): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" raise NotImplementedError - at cpython_api([{}], PyObject, borrowed=True) + at cpython_api([], PyObject, borrowed=True) def PyEval_GetGlobals(space, ): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" raise NotImplementedError - at cpython_api([{}], {PyFrameObject*}, borrowed=True) + at cpython_api([], {PyFrameObject*}, borrowed=True) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is currently executing.""" @@ -4848,7 +4759,7 @@ """Return the line number that frame is currently executing.""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyEval_GetRestricted(space, ): """If there is a current frame and it is executing in restricted mode, return true, otherwise false.""" @@ -5020,7 +4931,7 @@ equivalent to the Python expression tuple(o).""" raise NotImplementedError - at cpython_api([PyObject], {PyObject**}) + at cpython_api([PyObject], PyObjectP) def PySequence_Fast_ITEMS(space, o): """Return the underlying array of PyObject pointers. Assumes that o was returned by PySequence_Fast() and o is not NULL. @@ -5212,13 +5123,6 @@ systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyString_CheckExact(space, o): - """Return true if the object o is a string object, but not an instance of a - subtype of the string type. - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, ...], PyObject) def PyString_FromFormat(space, format, ): """Take a C printf()-style format string and a variable number of @@ -5408,7 +5312,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{PyObject**}, PyObject], lltype.Void) + at cpython_api([PyObjectP, PyObject], lltype.Void) def PyString_Concat(space, string, newpart): """Create a new string object in *string containing the contents of newpart appended to string; the caller will own the new reference. The reference to @@ -5417,35 +5321,19 @@ *string will be set to NULL; the appropriate exception will be set.""" raise NotImplementedError - at cpython_api([{PyObject**}, PyObject], lltype.Void) + at cpython_api([PyObjectP, PyObject], lltype.Void) def PyString_ConcatAndDel(space, string, newpart): """Create a new string object in *string containing the contents of newpart appended to string. This version decrements the reference count of newpart.""" raise NotImplementedError - at cpython_api([{PyObject**}, Py_ssize_t], rffi.INT_real) -def _PyString_Resize(space, string, newsize): - """A way to resize a string object even though it is "immutable". Only use this to - build up a brand new string object; don't use this if the string may already be - known in other parts of the code. It is an error to call this function if the - refcount on the input string object is not one. Pass the address of an existing - string object as an lvalue (it may be written into), and the new size desired. - On success, *string holds the resized string object and 0 is returned; - the address in *string may differ from its input value. If the reallocation - fails, the original string object at *string is deallocated, *string is - set to NULL, a memory exception is set, and -1 is returned. - - This function used an int type for newsize. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PyString_Format(space, format, args): """Return a new string object from format and args. Analogous to format % args. The args argument must be a tuple.""" raise NotImplementedError - at cpython_api([{PyObject**}], lltype.Void) + at cpython_api([PyObjectP], lltype.Void) def PyString_InternInPlace(space, string): """Intern the argument *string in place. The argument must be the address of a pointer variable pointing to a Python string object. If there is an existing @@ -5537,7 +5425,7 @@ one of the strings '' or '???'.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PyOS_AfterFork(space, ): """Function to update some internal state after a process fork; this should be called in the new process if the Python interpreter will continue to be used. @@ -5545,7 +5433,7 @@ to be called.""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyOS_CheckStack(space, ): """Return true when the interpreter runs out of stack space. This is a reliable check, but is only available when USE_STACKCHECK is defined (currently @@ -5590,7 +5478,7 @@ on error.""" raise NotImplementedError - at cpython_api([{}], lltype.Void) + at cpython_api([], lltype.Void) def PySys_ResetWarnOptions(space, ): """Reset sys.warnoptions to an empty list.""" raise NotImplementedError @@ -5668,13 +5556,6 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyTuple_CheckExact(space, p): - """Return true if p is a tuple object, but not an instance of a subtype of the - tuple type. - """ - raise NotImplementedError - @cpython_api([Py_ssize_t, ...], PyObject) def PyTuple_Pack(space, n, ): """Return a new tuple object of size n, or NULL on failure. The tuple values @@ -5695,15 +5576,6 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyTuple_GET_SIZE(space, p): - """Return the size of the tuple p, which must be non-NULL and point to a tuple; - no error checking is performed. - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) def PyTuple_GET_ITEM(space, p, pos): """Like PyTuple_GetItem(), but does no checking of its arguments. @@ -5732,7 +5604,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{PyObject**}, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real) def _PyTuple_Resize(space, p, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there @@ -5751,7 +5623,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyTuple_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ @@ -5770,7 +5642,7 @@ """ raise NotImplementedError - at cpython_api([{}], {unsigned int}) + at cpython_api([], {unsigned int}) def PyType_ClearCache(space, ): """Clear the internal lookup cache. Return the current version tag. """ @@ -5860,7 +5732,7 @@ PyUnicodeObject (not checked).""" raise NotImplementedError - at cpython_api([{}], rffi.INT_real) + at cpython_api([], rffi.INT_real) def PyUnicode_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ @@ -6766,7 +6638,7 @@ The other arguments are set to NULL.""" raise NotImplementedError - at cpython_api([{PyCodeObject*}, PyObject, PyObject, {PyObject**}, rffi.INT_real, {PyObject**}, rffi.INT_real, {PyObject**}, rffi.INT_real, PyObject], PyObject) + at cpython_api([{PyCodeObject*}, PyObject, PyObject, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObject], PyObject) def PyEval_EvalCodeEx(space, co, globals, locals, args, argcount, kws, kwcount, defs, defcount, closure): """Evaluate a precompiled code object, given a particular environment for its evaluation. This environment consists of dictionaries of global and local From benjamin at codespeak.net Sat Apr 3 22:46:50 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Apr 2010 22:46:50 +0200 (CEST) Subject: [pypy-svn] r73342 - pypy/branch/improve-jit-interp Message-ID: <20100403204650.7AF0A282B90@codespeak.net> Author: benjamin Date: Sat Apr 3 22:46:49 2010 New Revision: 73342 Removed: pypy/branch/improve-jit-interp/ Log: remove branch that went nowhere From benjamin at codespeak.net Sun Apr 4 01:01:05 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 4 Apr 2010 01:01:05 +0200 (CEST) Subject: [pypy-svn] r73343 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100403230105.46900282B9C@codespeak.net> Author: benjamin Date: Sun Apr 4 01:01:03 2010 New Revision: 73343 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: (pitrou) python 2.5 can raise an OverflowError computing MAX_SIZE Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Sun Apr 4 01:01:03 2010 @@ -114,8 +114,17 @@ assert max_n >= 0 ITEM = A.OF ctypes_item = get_ctypes_type(ITEM, delayed_builders) - MAX_SIZE = sys.maxint/64 - PtrType = ctypes.POINTER(MAX_SIZE * ctypes_item) + # Python 2.5 ctypes can raise OverflowError on 64-bit builds + for n in [sys.maxint, 2**31]: + MAX_SIZE = n/64 + try: + PtrType = ctypes.POINTER(MAX_SIZE * ctypes_item) + except OverflowError, e: + pass + else: + break + else: + raise e class CArray(ctypes.Structure): if not A._hints.get('nolength'): From benjamin at codespeak.net Sun Apr 4 01:11:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 4 Apr 2010 01:11:48 +0200 (CEST) Subject: [pypy-svn] r73344 - in pypy/branch/cpython-extension: . dotviewer lib-python pypy pypy/interpreter pypy/interpreter/test pypy/jit/backend/x86/test pypy/jit/metainterp pypy/module/_locale/test pypy/module/_ssl pypy/module/exceptions pypy/module/pyexpat pypy/module/pyexpat/test pypy/objspace/flow pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/tool pypy/translator/c pypy/translator/c/test pypy/translator/platform Message-ID: <20100403231148.E90FD282B9C@codespeak.net> Author: benjamin Date: Sun Apr 4 01:11:46 2010 New Revision: 73344 Modified: pypy/branch/cpython-extension/ (props changed) pypy/branch/cpython-extension/dotviewer/ (props changed) pypy/branch/cpython-extension/lib-python/ (props changed) pypy/branch/cpython-extension/pypy/ (props changed) pypy/branch/cpython-extension/pypy/interpreter/function.py pypy/branch/cpython-extension/pypy/interpreter/gateway.py pypy/branch/cpython-extension/pypy/interpreter/pyframe.py pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py pypy/branch/cpython-extension/pypy/interpreter/test/test_function.py pypy/branch/cpython-extension/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/branch/cpython-extension/pypy/jit/metainterp/logger.py (props changed) pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py pypy/branch/cpython-extension/pypy/module/_ssl/interp_ssl.py pypy/branch/cpython-extension/pypy/module/exceptions/ (props changed) pypy/branch/cpython-extension/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/cpython-extension/pypy/module/pyexpat/test/test_build.py pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py pypy/branch/cpython-extension/pypy/objspace/std/objspace.py pypy/branch/cpython-extension/pypy/objspace/std/register_all.py pypy/branch/cpython-extension/pypy/objspace/std/test/test_setobject.py (props changed) pypy/branch/cpython-extension/pypy/rlib/rcoroutine.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/cpython-extension/pypy/tool/runsubprocess.py pypy/branch/cpython-extension/pypy/translator/c/database.py pypy/branch/cpython-extension/pypy/translator/c/test/test_refcount.py (props changed) pypy/branch/cpython-extension/pypy/translator/platform/__init__.py pypy/branch/cpython-extension/pypy/translator/platform/maemo.py pypy/branch/cpython-extension/pypy/translator/platform/posix.py pypy/branch/cpython-extension/pypy/translator/platform/windows.py Log: merge from trunk Modified: pypy/branch/cpython-extension/pypy/interpreter/function.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/function.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/function.py Sun Apr 4 01:11:46 2010 @@ -295,8 +295,13 @@ def descr_function__setstate__(self, space, w_args): from pypy.interpreter.pycode import PyCode args_w = space.unpackiterable(w_args) - (w_name, w_doc, w_code, w_func_globals, w_closure, w_defs_w, - w_func_dict, w_module) = args_w + try: + (w_name, w_doc, w_code, w_func_globals, w_closure, w_defs_w, + w_func_dict, w_module) = args_w + except ValueError: + # wrong args + raise OperationError(space.w_ValueError, + space.wrap("Wrong arguments to function.__setstate__")) self.space = space self.name = space.str_w(w_name) Modified: pypy/branch/cpython-extension/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/gateway.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/gateway.py Sun Apr 4 01:11:46 2010 @@ -576,6 +576,8 @@ except rstackovf.StackOverflow, e: rstackovf.check_stack_overflow() raise space.prebuilt_recursion_error + except RuntimeError: # not on top of py.py + raise OperationError(space.w_RuntimeError, space.w_None) # (verbose) performance hack below Modified: pypy/branch/cpython-extension/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyframe.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyframe.py Sun Apr 4 01:11:46 2010 @@ -7,18 +7,18 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter import pytraceback -import opcode from pypy.rlib.objectmodel import we_are_translated, instantiate from pypy.rlib.jit import hint from pypy.rlib.debug import make_sure_not_resized from pypy.rlib import jit +from pypy.tool import stdlib_opcode # Define some opcodes used g = globals() for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY POP_BLOCK END_FINALLY'''.split(): - g[op] = opcode.opmap[op] -HAVE_ARGUMENT = opcode.HAVE_ARGUMENT + g[op] = stdlib_opcode.opmap[op] +HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT class PyFrame(eval.Frame): """Represents a frame for a regular Python function @@ -541,7 +541,7 @@ if delta_iblock < min_delta_iblock: min_delta_iblock = delta_iblock - if op >= opcode.HAVE_ARGUMENT: + if op >= stdlib_opcode.HAVE_ARGUMENT: addr += 3 else: addr += 1 Modified: pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py Sun Apr 4 01:11:46 2010 @@ -7,18 +7,16 @@ import sys from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import UnpackValueError, Wrappable -from pypy.interpreter import gateway, function, eval -from pypy.interpreter import pyframe, pytraceback +from pypy.interpreter import gateway, function, eval, pyframe, pytraceback from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib import jit +from pypy.rlib import jit, rstackovf from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT -from pypy.tool.stdlib_opcode import unrolling_opcode_descs -from pypy.tool.stdlib_opcode import opcode_method_names from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib import rstackovf +from pypy.tool.stdlib_opcode import (opcodedesc, HAVE_ARGUMENT, + unrolling_opcode_descs, + opcode_method_names) def unaryoperation(operationname): """NOT_RPYTHON""" Modified: pypy/branch/cpython-extension/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/test/test_function.py Sun Apr 4 01:11:46 2010 @@ -288,6 +288,11 @@ f = lambda: 42 assert f.func_doc is None + def test_setstate_called_with_wrong_args(self): + f = lambda: 42 + # not sure what it should raise, since CPython doesn't have setstate + # on function types + raises(ValueError, type(f).__setstate__, f, (1, 2, 3)) class AppTestMethod: def test_simple_call(self): Modified: pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py (original) +++ pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py Sun Apr 4 01:11:46 2010 @@ -22,8 +22,15 @@ current = _locale.setlocale(_locale.LC_ALL) try: try: - _locale.setlocale(_locale.LC_ALL, - space.str_w(cls.w_language_en)) + # some systems are only UTF-8 oriented + try: + _locale.setlocale(_locale.LC_ALL, + space.str_w(cls.w_language_en)) + except _locale.Error: + _locale.setlocale(_locale.LC_ALL, + space.str_w(cls.w_language_utf8)) + cls.w_language_en = cls.w_language_utf8 + _locale.setlocale(_locale.LC_ALL, space.str_w(cls.w_language_pl)) except _locale.Error: @@ -111,10 +118,11 @@ assert string.lowercase == lcase assert string.uppercase == ucase - _locale.setlocale(_locale.LC_ALL, self.language_en) + if self.language_en != self.language_utf8: + _locale.setlocale(_locale.LC_ALL, self.language_en) - assert string.lowercase != lcase - assert string.uppercase != ucase + assert string.lowercase != lcase + assert string.uppercase != ucase def test_localeconv(self): import _locale Modified: pypy/branch/cpython-extension/pypy/module/_ssl/interp_ssl.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_ssl/interp_ssl.py (original) +++ pypy/branch/cpython-extension/pypy/module/_ssl/interp_ssl.py Sun Apr 4 01:11:46 2010 @@ -18,10 +18,11 @@ # need of winsock2. Remove this when separate compilation is # available... 'winsock2.h', - 'openssl/ssl.h'] + 'openssl/ssl.h', + 'openssl/err.h'] else: libraries = ['ssl', 'crypto'] - includes = ['openssl/ssl.h'] + includes = ['openssl/ssl.h', 'openssl/err.h'] eci = ExternalCompilationInfo( libraries = libraries, @@ -140,7 +141,7 @@ ssl_external('SSL_get_error', [SSL_P, rffi.INT], rffi.INT) ssl_external('ERR_get_error', [], rffi.INT) -ssl_external('ERR_error_string', [rffi.INT, rffi.CCHARP], rffi.CCHARP) +ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP) ssl_external('SSL_get_peer_certificate', [SSL_P], X509_P) ssl_external('X509_get_subject_name', [X509_P], X509_NAME_P) ssl_external('X509_get_issuer_name', [X509_P], X509_NAME_P) Modified: pypy/branch/cpython-extension/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/cpython-extension/pypy/module/pyexpat/interp_pyexpat.py Sun Apr 4 01:11:46 2010 @@ -33,7 +33,7 @@ ]) XML_Content_Ptr = lltype.Ptr(lltype.ForwardReference()) -XML_Parser = rffi.VOIDP # an opaque pointer +XML_Parser = rffi.COpaquePtr(typedef='XML_Parser') class CConfigure: _compilation_info_ = eci @@ -475,7 +475,7 @@ global_storage.get_nonmoving_id( CallbackData(space, parser), id=rffi.cast(lltype.Signed, parser.itself)) - XML_SetUserData(parser.itself, parser.itself) + XML_SetUserData(parser.itself, rffi.cast(rffi.VOIDP, parser.itself)) # copy handlers from self for i in range(NB_HANDLERS): @@ -621,7 +621,7 @@ global_storage.get_nonmoving_id( CallbackData(space, parser), id=rffi.cast(lltype.Signed, parser.itself)) - XML_SetUserData(parser.itself, parser.itself) + XML_SetUserData(parser.itself, rffi.cast(rffi.VOIDP, parser.itself)) return space.wrap(parser) ParserCreate.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] Modified: pypy/branch/cpython-extension/pypy/module/pyexpat/test/test_build.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/pyexpat/test/test_build.py (original) +++ pypy/branch/cpython-extension/pypy/module/pyexpat/test/test_build.py Sun Apr 4 01:11:46 2010 @@ -18,6 +18,8 @@ def test_build(): def entry_point(argv): + parser = interp_pyexpat.XML_ParserCreate("test") + interp_pyexpat.XML_ParserFree(parser) res = interp_pyexpat.XML_ErrorString(3) os.write(1, rffi.charp2str(res)) return 0 Modified: pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py Sun Apr 4 01:11:46 2010 @@ -457,7 +457,7 @@ # XXX same as w_KeyboardInterrupt() raise RuntimeError("the interpreter raises RuntimeError during " "flow graph construction") - w_RuntimeError = property(w_RuntimeError) + w_RuntimeError = prebuilt_recursion_error = property(w_RuntimeError) # the following gives us easy access to declare more for applications: NOT_REALLY_CONST = { Modified: pypy/branch/cpython-extension/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/objspace.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/objspace.py Sun Apr 4 01:11:46 2010 @@ -89,7 +89,6 @@ def _install_multimethods(self): """Install all the MultiMethods into the space instance.""" - model.add_extra_comparisons() for name, mm in model.MM.__dict__.items(): if not isinstance(mm, model.StdObjSpaceMultiMethod): continue Modified: pypy/branch/cpython-extension/pypy/objspace/std/register_all.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/register_all.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/register_all.py Sun Apr 4 01:11:46 2010 @@ -44,6 +44,8 @@ func = hack_func_by_name(funcname, namespaces) func.register(obj, *l) + model.add_extra_comparisons() + def hack_func_by_name(funcname, namespaces): for ns in namespaces: Modified: pypy/branch/cpython-extension/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/rcoroutine.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/rcoroutine.py Sun Apr 4 01:11:46 2010 @@ -111,7 +111,7 @@ self.temp_exc = exc def check_for_zombie(self, obj): - return co in self.to_delete + return obj in self.to_delete def postpone_deletion(self, obj): self.to_delete.append(obj) Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py Sun Apr 4 01:11:46 2010 @@ -114,8 +114,17 @@ assert max_n >= 0 ITEM = A.OF ctypes_item = get_ctypes_type(ITEM, delayed_builders) - MAX_SIZE = sys.maxint/64 - PtrType = ctypes.POINTER(MAX_SIZE * ctypes_item) + # Python 2.5 ctypes can raise OverflowError on 64-bit builds + for n in [sys.maxint, 2**31]: + MAX_SIZE = n/64 + try: + PtrType = ctypes.POINTER(MAX_SIZE * ctypes_item) + except OverflowError, e: + pass + else: + break + else: + raise e class CArray(ctypes.Structure): if not A._hints.get('nolength'): Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py Sun Apr 4 01:11:46 2010 @@ -449,7 +449,7 @@ return lltype.Ptr(lltype.FuncType(args, res)) CCallback._annspecialcase_ = 'specialize:memo' -def COpaque(name, hints=None, compilation_info=None): +def COpaque(name=None, ptr_typedef=None, hints=None, compilation_info=None): if compilation_info is None: compilation_info = ExternalCompilationInfo() if hints is None: @@ -457,7 +457,10 @@ else: hints = hints.copy() hints['external'] = 'C' - hints['c_name'] = name + if name is not None: + hints['c_name'] = name + if ptr_typedef is not None: + hints['c_pointer_typedef'] = ptr_typedef def lazy_getsize(cache={}): from pypy.rpython.tool import rffi_platform try: @@ -471,7 +474,8 @@ return lltype.OpaqueType(name, hints) def COpaquePtr(*args, **kwds): - return lltype.Ptr(COpaque(*args, **kwds)) + typedef = kwds.pop('typedef', None) + return lltype.Ptr(COpaque(ptr_typedef=typedef, *args, **kwds)) def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant, sandboxsafe=False, _nowrapper=False, Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_rffi.py Sun Apr 4 01:11:46 2010 @@ -280,6 +280,28 @@ f1 = self.compile(f, []) assert f1() == 'a' + def test_opaque_typedef(self): + code = """ + #include + struct stuff; + typedef struct stuff *stuff_ptr; + static int get(stuff_ptr ptr) { return (ptr != NULL); } + """ + + eci = ExternalCompilationInfo( + post_include_bits = [code] + ) + + STUFFP = COpaquePtr(typedef='stuff_ptr', compilation_info=eci) + ll_get = llexternal('get', [STUFFP], lltype.Signed, + compilation_info=eci) + + def f(): + return ll_get(lltype.nullptr(STUFFP.TO)) + + f1 = self.compile(f, []) + assert f1() == 0 + def return_char(self, signed): ctype_pref = ["un", ""][signed] rffi_type = [UCHAR, SIGNEDCHAR][signed] Modified: pypy/branch/cpython-extension/pypy/tool/runsubprocess.py ============================================================================== --- pypy/branch/cpython-extension/pypy/tool/runsubprocess.py (original) +++ pypy/branch/cpython-extension/pypy/tool/runsubprocess.py Sun Apr 4 01:11:46 2010 @@ -47,8 +47,8 @@ _source = os.path.dirname(os.path.abspath(__file__)) _source = os.path.join(_source, 'runsubprocess.py') # and not e.g. '.pyc' # Let's not hear about os.popen2's deprecation. - warnings.filterwarnings("ignore", "popen2", DeprecationWarning, - "runsubprocess") + warnings.filterwarnings("ignore", ".*popen2.*", DeprecationWarning, + "pypy.tool.runsubprocess") _child_stdin, _child_stdout = os.popen2( "'%s' '%s'" % (sys.executable, _source)) Modified: pypy/branch/cpython-extension/pypy/translator/c/database.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/database.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/database.py Sun Apr 4 01:11:46 2010 @@ -101,6 +101,9 @@ if isinstance(T, Primitive) or T == GCREF: return PrimitiveType[T] elif isinstance(T, Ptr): + if (isinstance(T.TO, OpaqueType) and + T.TO.hints.get('c_pointer_typedef') is not None): + return '%s @' % T.TO.hints['c_pointer_typedef'] try: node = self.gettypedefnode(T.TO) except NoCorrespondingNode: Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Sun Apr 4 01:11:46 2010 @@ -114,9 +114,12 @@ for line in stderr.splitlines(): log.WARNING(line) - + def _preprocess_include_dirs(self, include_dirs): + # hook for maemo + return include_dirs + def _compile_args_from_eci(self, eci, standalone): - include_dirs = self._preprocess_dirs(eci.include_dirs) + include_dirs = self._preprocess_include_dirs(eci.include_dirs) args = self._includedirs(include_dirs) if standalone: extra = self.standalone_only Modified: pypy/branch/cpython-extension/pypy/translator/platform/maemo.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/maemo.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/maemo.py Sun Apr 4 01:11:46 2010 @@ -42,7 +42,7 @@ self.copied_cache[dir_from] = new_dirpath return new_dirpath - def _preprocess_dirs(self, include_dirs): + def _preprocess_include_dirs(self, include_dirs): """ Tweak includedirs so they'll be available through scratchbox """ res_incl_dirs = [] Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/posix.py Sun Apr 4 01:11:46 2010 @@ -45,10 +45,6 @@ cwd=str(exe_name.dirpath())) return exe_name - def _preprocess_dirs(self, include_dirs): - # hook for maemo - return include_dirs - def _pkg_config(self, lib, opt, default): try: ret, out, err = _run_subprocess("pkg-config", [lib, opt]) @@ -88,7 +84,7 @@ m.cfiles = rel_cfiles rel_includedirs = [pypyrel(incldir) for incldir in - self._preprocess_dirs(eci.include_dirs)] + self._preprocess_include_dirs(eci.include_dirs)] m.comment('automatically generated makefile') definitions = [ Modified: pypy/branch/cpython-extension/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/windows.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/windows.py Sun Apr 4 01:11:46 2010 @@ -116,17 +116,6 @@ # The following symbol is used in c/src/stack.h self.cflags.append('/DMAX_STACK_SIZE=%d' % (stack_size - 1024)) - if hasattr(sys, 'exec_prefix'): - self.add_cpython_dirs = True - else: - # We are certainly running pypy-c - self.add_cpython_dirs = False - - def _preprocess_dirs(self, include_dirs): - if self.add_cpython_dirs: - return include_dirs + (py.path.local(sys.exec_prefix).join('PC'),) - return include_dirs - def _includedirs(self, include_dirs): return ['/I%s' % (idir,) for idir in include_dirs] From trundle at codespeak.net Sun Apr 4 01:59:20 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Sun, 4 Apr 2010 01:59:20 +0200 (CEST) Subject: [pypy-svn] r73345 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100403235920.68263282B9C@codespeak.net> Author: trundle Date: Sun Apr 4 01:59:18 2010 New Revision: 73345 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Add T_CHAR member. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Sun Apr 4 01:59:18 2010 @@ -24,6 +24,7 @@ #define T_INT 1 #define T_STRING 5 #define T_OBJECT 6 +#define T_CHAR 7 /* 1-character string */ #define T_STRING_INPLACE 13 /* Strings contained in the structure */ #define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError when the value is NULL, instead of Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Sun Apr 4 01:59:18 2010 @@ -6,7 +6,8 @@ from pypy.module.cpyext.intobject import PyInt_AsLong from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, from_ref, make_ref -from pypy.module.cpyext.stringobject import PyString_FromString +from pypy.module.cpyext.stringobject import (PyString_FromString, + PyString_FromStringAndSize) from pypy.module.cpyext.typeobjectdefs import PyMemberDef @@ -27,6 +28,9 @@ elif member_type == structmemberdefs.T_STRING_INPLACE: result = rffi.cast(rffi.CCHARP, addr) w_result = PyString_FromString(space, result) + elif member_type == structmemberdefs.T_CHAR: + result = rffi.cast(rffi.CCHARP, addr) + w_result = space.wrap(result[0]) elif member_type in [structmemberdefs.T_OBJECT, structmemberdefs.T_OBJECT_EX]: obj_ptr = rffi.cast(PyObjectP, addr) @@ -68,6 +72,13 @@ w_long_value = PyInt_AsLong(space, w_value) array = rffi.cast(rffi.INTP, addr) array[0] = rffi.cast(rffi.INT, w_long_value) + elif member_type == structmemberdefs.T_CHAR: + str_value = space.str_w(w_value) + if len(str_value) != 1: + raise OperationError(space.w_TypeError, + space.wrap("string of length 1 expected")) + array = rffi.cast(rffi.CCHARP, addr) + array[0] = str_value elif member_type in [structmemberdefs.T_OBJECT, structmemberdefs.T_OBJECT_EX]: array = rffi.cast(PyObjectP, addr) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Sun Apr 4 01:59:18 2010 @@ -1,6 +1,7 @@ T_INT = 1 T_STRING = 5 T_OBJECT = 6 +T_CHAR = 7 T_STRING_INPLACE = 13 T_OBJECT_EX = 16 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Sun Apr 4 01:59:18 2010 @@ -121,6 +121,7 @@ "A string."}, {"string_member_inplace", T_STRING_INPLACE, offsetof(fooobject, foo_string_inplace), 0, "An inplace string."}, + {"char_member", T_CHAR, offsetof(fooobject, foo_string_inplace), 0, NULL}, {NULL} /* Sentinel */ }; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 01:59:18 2010 @@ -64,6 +64,11 @@ assert obj.string_member_inplace == "spam" raises(TypeError, "obj.string_member_inplace = 42") raises(TypeError, "del obj.string_member_inplace") + assert obj.char_member == "s" + obj.char_member = "a" + assert obj.char_member == "a" + raises(TypeError, "obj.char_member = 'spam'") + raises(TypeError, "obj.char_member = 42") #skip("In progress") # not at all, how does this fail for you amaury? From trundle at codespeak.net Sun Apr 4 02:50:52 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Sun, 4 Apr 2010 02:50:52 +0200 (CEST) Subject: [pypy-svn] r73346 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404005052.73F38282B9C@codespeak.net> Author: trundle Date: Sun Apr 4 02:50:50 2010 New Revision: 73346 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Log: rpythonify T_CHAR Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Sun Apr 4 02:50:50 2010 @@ -78,7 +78,7 @@ raise OperationError(space.w_TypeError, space.wrap("string of length 1 expected")) array = rffi.cast(rffi.CCHARP, addr) - array[0] = str_value + array[0] = str_value[0] elif member_type in [structmemberdefs.T_OBJECT, structmemberdefs.T_OBJECT_EX]: array = rffi.cast(PyObjectP, addr) From benjamin at codespeak.net Sun Apr 4 04:43:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 4 Apr 2010 04:43:45 +0200 (CEST) Subject: [pypy-svn] r73347 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100404024345.B96CC282BEC@codespeak.net> Author: benjamin Date: Sun Apr 4 04:43:43 2010 New Revision: 73347 Modified: pypy/trunk/pypy/annotation/bookkeeper.py pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py Log: allow constant methods to have their attributes taken Modified: pypy/trunk/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/trunk/pypy/annotation/bookkeeper.py (original) +++ pypy/trunk/pypy/annotation/bookkeeper.py Sun Apr 4 04:43:43 2010 @@ -523,7 +523,7 @@ self.getdesc(pyobj.im_func), # funcdesc self.getuniqueclassdef(origincls), # originclassdef classdef, # selfclassdef - name) + name, pyobj=pyobj) else: # must be a frozen pre-built constant, but let's check try: @@ -552,7 +552,7 @@ return result def getmethoddesc(self, funcdesc, originclassdef, selfclassdef, name, - flags={}): + flags={}, pyobj=None): flagskey = flags.items() flagskey.sort() key = funcdesc, originclassdef, selfclassdef, name, tuple(flagskey) @@ -560,7 +560,7 @@ return self.methoddescs[key] except KeyError: result = description.MethodDesc(self, funcdesc, originclassdef, - selfclassdef, name, flags) + selfclassdef, name, flags, pyobj) self.methoddescs[key] = result return result Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sun Apr 4 04:43:43 2010 @@ -689,8 +689,8 @@ knowntype = types.MethodType def __init__(self, bookkeeper, funcdesc, originclassdef, - selfclassdef, name, flags={}): - super(MethodDesc, self).__init__(bookkeeper) + selfclassdef, name, flags={}, pyobj=None): + super(MethodDesc, self).__init__(bookkeeper, pyobj) self.funcdesc = funcdesc self.originclassdef = originclassdef self.selfclassdef = selfclassdef Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Sun Apr 4 04:43:43 2010 @@ -1751,6 +1751,23 @@ s = a.build_types(f, [bool]) assert s == annmodel.SomeString(can_be_None=True) + def test_method_hasattr(self): + class X: + def m(self): + return 4 + m.attr = 23 + def x(self, string): + return string + x = X() + m = x.m + def f(string): + if hasattr(m, "attr"): + return x.x(string) + return x.m() + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert s == annmodel.SomeString() + def test_dont_see_AttributeError_clause(self): class Stuff: def _freeze_(self): From benjamin at codespeak.net Sun Apr 4 05:43:33 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 4 Apr 2010 05:43:33 +0200 (CEST) Subject: [pypy-svn] r73348 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100404034333.D2D20282BD8@codespeak.net> Author: benjamin Date: Sun Apr 4 05:43:31 2010 New Revision: 73348 Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py Log: propogate method constantness to bound methods Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sun Apr 4 05:43:31 2010 @@ -293,7 +293,8 @@ return self.bookkeeper.getmethoddesc(self, classdef, # originclassdef, None, # selfclassdef - name) + name, + pyobj=self.pyobj) def consider_call_site(bookkeeper, family, descs, args, s_result): shape = rawshape(args) @@ -727,7 +728,8 @@ self.originclassdef, newselfclassdef, self.name, - flags) + flags, + self.pyobj) def consider_call_site(bookkeeper, family, descs, args, s_result): shape = rawshape(args, nextra=1) # account for the extra 'self' Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Sun Apr 4 05:43:31 2010 @@ -1768,6 +1768,22 @@ s = a.build_types(f, [str]) assert s == annmodel.SomeString() + def test_more_complex_method_hasattr(self): + class X: + def m(self): + return 4 + m.attr = 32 + def f(): + x = X() + if hasattr(x.m, "attr"): + return 3 + return 0 + a = self.RPythonAnnotator() + s = a.build_types(f, []) + s_res = annmodel.SomeInteger(True) + s_res.const = 3 + assert s == s_res + def test_dont_see_AttributeError_clause(self): class Stuff: def _freeze_(self): From benjamin at codespeak.net Sun Apr 4 05:56:15 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 4 Apr 2010 05:56:15 +0200 (CEST) Subject: [pypy-svn] r73349 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100404035615.5DE91282BEF@codespeak.net> Author: benjamin Date: Sun Apr 4 05:56:13 2010 New Revision: 73349 Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py Log: revert r74438; not well thought out Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sun Apr 4 05:56:13 2010 @@ -293,8 +293,7 @@ return self.bookkeeper.getmethoddesc(self, classdef, # originclassdef, None, # selfclassdef - name, - pyobj=self.pyobj) + name) def consider_call_site(bookkeeper, family, descs, args, s_result): shape = rawshape(args) @@ -728,8 +727,7 @@ self.originclassdef, newselfclassdef, self.name, - flags, - self.pyobj) + flags) def consider_call_site(bookkeeper, family, descs, args, s_result): shape = rawshape(args, nextra=1) # account for the extra 'self' Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Sun Apr 4 05:56:13 2010 @@ -1768,22 +1768,6 @@ s = a.build_types(f, [str]) assert s == annmodel.SomeString() - def test_more_complex_method_hasattr(self): - class X: - def m(self): - return 4 - m.attr = 32 - def f(): - x = X() - if hasattr(x.m, "attr"): - return 3 - return 0 - a = self.RPythonAnnotator() - s = a.build_types(f, []) - s_res = annmodel.SomeInteger(True) - s_res.const = 3 - assert s == s_res - def test_dont_see_AttributeError_clause(self): class Stuff: def _freeze_(self): From agaynor at codespeak.net Sun Apr 4 16:51:53 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 16:51:53 +0200 (CEST) Subject: [pypy-svn] r73356 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404145153.91E38282BF1@codespeak.net> Author: agaynor Date: Sun Apr 4 16:51:51 2010 New Revision: 73356 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: Implemented PyList_GET_SIZE Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Sun Apr 4 16:51:51 2010 @@ -45,3 +45,11 @@ PyErr_BadInternalCall(space) w_list.append(w_item) return 0 + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyList_GET_SIZE(space, w_list): + """Macro form of PyList_Size() without error checking. + + This macro returned an int. This might require changes in your + code for properly supporting 64-bit systems.""" + return len(w_list.wrappeditems) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 16:51:51 2010 @@ -3329,14 +3329,6 @@ your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyList_GET_SIZE(space, list): - """Macro form of PyList_Size() without error checking. - - This macro returned an int. This might require changes in your - code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) def PyList_GetItem(space, list, index): """Return the object at position pos in the list pointed to by p. The Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Sun Apr 4 16:51:51 2010 @@ -19,6 +19,12 @@ assert not api.PyList_Check(space.newtuple([])) assert not api.PyList_CheckExact(space.newtuple([])) + + def test_get_size(self, space, api): + l = api.PyList_New(0) + assert api.PyList_GET_SIZE(l) == 0 + api.PyList_Append(l, space.wrap(3)) + assert api.PyList_GET_SIZE(l) == 1 class AppTestListObject(AppTestCpythonExtensionBase): def test_listobject(self): From agaynor at codespeak.net Sun Apr 4 17:01:52 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 17:01:52 +0200 (CEST) Subject: [pypy-svn] r73357 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404150152.C6545282BF3@codespeak.net> Author: agaynor Date: Sun Apr 4 17:01:51 2010 New Revision: 73357 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Log: Implemented PyObject_CallObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/eval.py Sun Apr 4 17:01:51 2010 @@ -4,3 +4,13 @@ @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) + + at cpython_api([PyObject, PyObject], PyObject) +def PyObject_CallObject(space, w_obj, w_arg): + """ + Call a callable Python object callable_object, with arguments given by the + tuple args. If no arguments are needed, then args may be NULL. Returns + the result of the call on success, or NULL on failure. This is the equivalent + of the Python expression apply(callable_object, args) or + callable_object(*args).""" + return space.call(w_obj, w_arg) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 17:01:51 2010 @@ -4516,19 +4516,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyObject_CallObject(space, callable_object, args): - """ - - - - Call a callable Python object callable_object, with arguments given by the - tuple args. If no arguments are needed, then args may be NULL. Returns - the result of the call on success, or NULL on failure. This is the equivalent - of the Python expression apply(callable_object, args) or - callable_object(*args).""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, ...], PyObject) def PyObject_CallFunction(space, callable, format, ): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Sun Apr 4 17:01:51 2010 @@ -28,3 +28,30 @@ space.setitem(w_d, space.wrap("xyz"), space.wrap(3)) w_res = api.PyEval_CallObjectWithKeywords(w_f, w_t, w_d) assert space.int_w(w_res) == 21 + + def test_call_object(self, space, api): + w_l, w_f = space.fixedview(space.appexec([], """(): + l = [] + def f(arg1, arg2): + l.append(arg1) + l.append(arg2) + return len(l) + return l, f + """)) + + w_t = space.newtuple([space.wrap(1), space.wrap(2)]) + w_res = api.PyObject_CallObject(w_f, w_t) + assert space.int_w(w_res) == 2 + assert space.int_w(space.len(w_l)) == 2 + + w_f = space.appexec([], """(): + def f(*args): + assert isinstance(args, tuple) + return len(args) + 8 + return f + """) + + w_t = space.newtuple([space.wrap(1), space.wrap(2)]) + w_res = api.PyObject_CallObject(w_f, w_t) + + assert space.int_w(w_res) == 10 From agaynor at codespeak.net Sun Apr 4 17:34:53 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 17:34:53 +0200 (CEST) Subject: [pypy-svn] r73358 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404153453.49CDF282BF6@codespeak.net> Author: agaynor Date: Sun Apr 4 17:34:51 2010 New Revision: 73358 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Log: Implemented PySequence_GetSlice Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Sun Apr 4 17:34:51 2010 @@ -44,3 +44,11 @@ assert isinstance(w_obj, tupleobject.W_TupleObject) return len(w_obj.wrappeditems) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) +def PySequence_GetSlice(space, w_obj, w_start, w_end): + """Return the slice of sequence object o between i1 and i2, or NULL on + failure. This is the equivalent of the Python expression o[i1:i2]. + + This function used an int type for i1 and i2. This might + require changes in your code for properly supporting 64-bit systems.""" + return space.getslice(w_obj, w_start, w_end) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 17:34:51 2010 @@ -4820,15 +4820,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) -def PySequence_GetSlice(space, o, i1, i2): - """Return the slice of sequence object o between i1 and i2, or NULL on - failure. This is the equivalent of the Python expression o[i1:i2]. - - This function used an int type for i1 and i2. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real) def PySequence_SetItem(space, o, i, v): """Assign object v to the ith element of o. Returns -1 on failure. This Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Sun Apr 4 17:34:51 2010 @@ -29,3 +29,8 @@ assert exc.value.match(space, space.w_TypeError) assert space.str_w(exc.value.get_w_value(space)) == "message" rffi.free_charp(message) + + def test_get_slice(self, space, api): + w_t = space.wrap((1, 2, 3, 4, 5)) + assert space.unwrap(api.PySequence_GetSlice(w_t, space.wrap(2), space.wrap(4))) == (3, 4) + assert space.unwrap(api.PySequence_GetSlice(w_t, space.wrap(1), space.wrap(-1))) == (2, 3, 4) From agaynor at codespeak.net Sun Apr 4 19:12:01 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 19:12:01 +0200 (CEST) Subject: [pypy-svn] r73359 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404171201.9332F282B90@codespeak.net> Author: agaynor Date: Sun Apr 4 19:11:59 2010 New Revision: 73359 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Log: A few fixes to the methods I implemented. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Sun Apr 4 19:11:59 2010 @@ -52,4 +52,5 @@ This macro returned an int. This might require changes in your code for properly supporting 64-bit systems.""" + assert isinstance(w_list, W_ListObject) return len(w_list.wrappeditems) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Sun Apr 4 19:11:59 2010 @@ -45,10 +45,10 @@ return len(w_obj.wrappeditems) @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) -def PySequence_GetSlice(space, w_obj, w_start, w_end): +def PySequence_GetSlice(space, w_obj, start, end): """Return the slice of sequence object o between i1 and i2, or NULL on failure. This is the equivalent of the Python expression o[i1:i2]. This function used an int type for i1 and i2. This might require changes in your code for properly supporting 64-bit systems.""" - return space.getslice(w_obj, w_start, w_end) + return space.getslice(w_obj, space.wrap(start), space.wrap(end)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Sun Apr 4 19:11:59 2010 @@ -32,5 +32,5 @@ def test_get_slice(self, space, api): w_t = space.wrap((1, 2, 3, 4, 5)) - assert space.unwrap(api.PySequence_GetSlice(w_t, space.wrap(2), space.wrap(4))) == (3, 4) - assert space.unwrap(api.PySequence_GetSlice(w_t, space.wrap(1), space.wrap(-1))) == (2, 3, 4) + assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == (3, 4) + assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == (2, 3, 4) From xoraxax at codespeak.net Sun Apr 4 19:15:00 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:15:00 +0200 (CEST) Subject: [pypy-svn] r73360 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100404171500.AA93C282BF5@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:14:58 2010 New Revision: 73360 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/sre.h (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/sre_constants.h (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Add _sre.c module from CPython trunk. Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c Sun Apr 4 19:14:58 2010 @@ -0,0 +1,3909 @@ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * partial history: + * 1999-10-24 fl created (based on existing template matcher code) + * 2000-03-06 fl first alpha, sort of + * 2000-08-01 fl fixes for 1.6b1 + * 2000-08-07 fl use PyOS_CheckStack() if available + * 2000-09-20 fl added expand method + * 2001-03-20 fl lots of fixes for 2.1b2 + * 2001-04-15 fl export copyright as Python attribute, not global + * 2001-04-28 fl added __copy__ methods (work in progress) + * 2001-05-14 fl fixes for 1.5.2 compatibility + * 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis) + * 2001-10-18 fl fixed group reset issue (from Matthew Mueller) + * 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1 + * 2001-10-21 fl added sub/subn primitive + * 2001-10-24 fl added finditer primitive (for 2.2 only) + * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) + * 2002-11-09 fl fixed empty sub/subn return type + * 2003-04-18 mvl fully support 4-byte codes + * 2003-10-17 gn implemented non recursive scheme + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * This version of the SRE library can be redistributed under CNRI's + * Python 1.6 license. For any other use, please contact Secret Labs + * AB (info at pythonware.com). + * + * Portions of this engine have been developed in cooperation with + * CNRI. Hewlett-Packard provided funding for 1.6 integration and + * other compatibility work. + */ + +#ifndef SRE_RECURSIVE + +static char copyright[] = + " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "structmember.h" /* offsetof */ + +#include "sre.h" + +#include + +/* name of this module, minus the leading underscore */ +#if !defined(SRE_MODULE) +#define SRE_MODULE "sre" +#endif + +#define SRE_PY_MODULE "re" + +/* defining this one enables tracing */ +#undef VERBOSE + +#if PY_VERSION_HEX >= 0x01060000 +#if PY_VERSION_HEX < 0x02020000 || defined(Py_USING_UNICODE) +/* defining this enables unicode support (default under 1.6a1 and later) */ +#define HAVE_UNICODE +#endif +#endif + +/* -------------------------------------------------------------------- */ +/* optional features */ + +/* enables fast searching */ +#define USE_FAST_SEARCH + +/* enables aggressive inlining (always on for Visual C) */ +#undef USE_INLINE + +/* enables copy/deepcopy handling (work in progress) */ +#undef USE_BUILTIN_COPY + +#if PY_VERSION_HEX < 0x01060000 +#define PyObject_DEL(op) PyMem_DEL((op)) +#endif + +/* -------------------------------------------------------------------- */ + +#if defined(_MSC_VER) +#pragma optimize("agtw", on) /* doesn't seem to make much difference... */ +#pragma warning(disable: 4710) /* who cares if functions are not inlined ;-) */ +/* fastest possible local call under MSVC */ +#define LOCAL(type) static __inline type __fastcall +#elif defined(USE_INLINE) +#define LOCAL(type) static inline type +#else +#define LOCAL(type) static type +#endif + +/* error codes */ +#define SRE_ERROR_ILLEGAL -1 /* illegal opcode */ +#define SRE_ERROR_STATE -2 /* illegal state */ +#define SRE_ERROR_RECURSION_LIMIT -3 /* runaway recursion */ +#define SRE_ERROR_MEMORY -9 /* out of memory */ +#define SRE_ERROR_INTERRUPTED -10 /* signal handler raised exception */ + +#if defined(VERBOSE) +#define TRACE(v) printf v +#else +#define TRACE(v) +#endif + +/* -------------------------------------------------------------------- */ +/* search engine state */ + +/* default character predicates (run sre_chars.py to regenerate tables) */ + +#define SRE_DIGIT_MASK 1 +#define SRE_SPACE_MASK 2 +#define SRE_LINEBREAK_MASK 4 +#define SRE_ALNUM_MASK 8 +#define SRE_WORD_MASK 16 + +/* FIXME: this assumes ASCII. create tables in init_sre() instead */ + +static char sre_char_info[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 2, +2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, +25, 25, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, +0, 0, 16, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0 }; + +static char sre_char_lower[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, +10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, +27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, +44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, +61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, +108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, +122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, +106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, +120, 121, 122, 123, 124, 125, 126, 127 }; + +#define SRE_IS_DIGIT(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_DIGIT_MASK) : 0) +#define SRE_IS_SPACE(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_SPACE_MASK) : 0) +#define SRE_IS_LINEBREAK(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_LINEBREAK_MASK) : 0) +#define SRE_IS_ALNUM(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_ALNUM_MASK) : 0) +#define SRE_IS_WORD(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_WORD_MASK) : 0) + +static unsigned int sre_lower(unsigned int ch) +{ + return ((ch) < 128 ? (unsigned int)sre_char_lower[ch] : ch); +} + +/* locale-specific character predicates */ +/* !(c & ~N) == (c < N+1) for any unsigned c, this avoids + * warnings when c's type supports only numbers < N+1 */ +#define SRE_LOC_IS_DIGIT(ch) (!((ch) & ~255) ? isdigit((ch)) : 0) +#define SRE_LOC_IS_SPACE(ch) (!((ch) & ~255) ? isspace((ch)) : 0) +#define SRE_LOC_IS_LINEBREAK(ch) ((ch) == '\n') +#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? isalnum((ch)) : 0) +#define SRE_LOC_IS_WORD(ch) (SRE_LOC_IS_ALNUM((ch)) || (ch) == '_') + +static unsigned int sre_lower_locale(unsigned int ch) +{ + return ((ch) < 256 ? (unsigned int)tolower((ch)) : ch); +} + +/* unicode-specific character predicates */ + +#if defined(HAVE_UNICODE) + +#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch)) +#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) +#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch)) +#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch)) +#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_') + +static unsigned int sre_lower_unicode(unsigned int ch) +{ + return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch)); +} + +#endif + +LOCAL(int) +sre_category(SRE_CODE category, unsigned int ch) +{ + switch (category) { + + case SRE_CATEGORY_DIGIT: + return SRE_IS_DIGIT(ch); + case SRE_CATEGORY_NOT_DIGIT: + return !SRE_IS_DIGIT(ch); + case SRE_CATEGORY_SPACE: + return SRE_IS_SPACE(ch); + case SRE_CATEGORY_NOT_SPACE: + return !SRE_IS_SPACE(ch); + case SRE_CATEGORY_WORD: + return SRE_IS_WORD(ch); + case SRE_CATEGORY_NOT_WORD: + return !SRE_IS_WORD(ch); + case SRE_CATEGORY_LINEBREAK: + return SRE_IS_LINEBREAK(ch); + case SRE_CATEGORY_NOT_LINEBREAK: + return !SRE_IS_LINEBREAK(ch); + + case SRE_CATEGORY_LOC_WORD: + return SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_LOC_NOT_WORD: + return !SRE_LOC_IS_WORD(ch); + +#if defined(HAVE_UNICODE) + case SRE_CATEGORY_UNI_DIGIT: + return SRE_UNI_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_NOT_DIGIT: + return !SRE_UNI_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_SPACE: + return SRE_UNI_IS_SPACE(ch); + case SRE_CATEGORY_UNI_NOT_SPACE: + return !SRE_UNI_IS_SPACE(ch); + case SRE_CATEGORY_UNI_WORD: + return SRE_UNI_IS_WORD(ch); + case SRE_CATEGORY_UNI_NOT_WORD: + return !SRE_UNI_IS_WORD(ch); + case SRE_CATEGORY_UNI_LINEBREAK: + return SRE_UNI_IS_LINEBREAK(ch); + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + return !SRE_UNI_IS_LINEBREAK(ch); +#else + case SRE_CATEGORY_UNI_DIGIT: + return SRE_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_NOT_DIGIT: + return !SRE_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_SPACE: + return SRE_IS_SPACE(ch); + case SRE_CATEGORY_UNI_NOT_SPACE: + return !SRE_IS_SPACE(ch); + case SRE_CATEGORY_UNI_WORD: + return SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_UNI_NOT_WORD: + return !SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_UNI_LINEBREAK: + return SRE_IS_LINEBREAK(ch); + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + return !SRE_IS_LINEBREAK(ch); +#endif + } + return 0; +} + +/* helpers */ + +static void +data_stack_dealloc(SRE_STATE* state) +{ + if (state->data_stack) { + PyMem_FREE(state->data_stack); + state->data_stack = NULL; + } + state->data_stack_size = state->data_stack_base = 0; +} + +static int +data_stack_grow(SRE_STATE* state, Py_ssize_t size) +{ + Py_ssize_t minsize, cursize; + minsize = state->data_stack_base+size; + cursize = state->data_stack_size; + if (cursize < minsize) { + void* stack; + cursize = minsize+minsize/4+1024; + TRACE(("allocate/grow stack %d\n", cursize)); + stack = PyMem_REALLOC(state->data_stack, cursize); + if (!stack) { + data_stack_dealloc(state); + return SRE_ERROR_MEMORY; + } + state->data_stack = (char *)stack; + state->data_stack_size = cursize; + } + return 0; +} + +/* generate 8-bit version */ + +#define SRE_CHAR unsigned char +#define SRE_AT sre_at +#define SRE_COUNT sre_count +#define SRE_CHARSET sre_charset +#define SRE_INFO sre_info +#define SRE_MATCH sre_match +#define SRE_MATCH_CONTEXT sre_match_context +#define SRE_SEARCH sre_search +#define SRE_LITERAL_TEMPLATE sre_literal_template + +#if defined(HAVE_UNICODE) + +#define SRE_RECURSIVE +#include "_sre.c" +#undef SRE_RECURSIVE + +#undef SRE_LITERAL_TEMPLATE +#undef SRE_SEARCH +#undef SRE_MATCH +#undef SRE_MATCH_CONTEXT +#undef SRE_INFO +#undef SRE_CHARSET +#undef SRE_COUNT +#undef SRE_AT +#undef SRE_CHAR + +/* generate 16-bit unicode version */ + +#define SRE_CHAR Py_UNICODE +#define SRE_AT sre_uat +#define SRE_COUNT sre_ucount +#define SRE_CHARSET sre_ucharset +#define SRE_INFO sre_uinfo +#define SRE_MATCH sre_umatch +#define SRE_MATCH_CONTEXT sre_umatch_context +#define SRE_SEARCH sre_usearch +#define SRE_LITERAL_TEMPLATE sre_uliteral_template +#endif + +#endif /* SRE_RECURSIVE */ + +/* -------------------------------------------------------------------- */ +/* String matching engine */ + +/* the following section is compiled twice, with different character + settings */ + +LOCAL(int) +SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) +{ + /* check if pointer is at given position */ + + Py_ssize_t thisp, thatp; + + switch (at) { + + case SRE_AT_BEGINNING: + case SRE_AT_BEGINNING_STRING: + return ((void*) ptr == state->beginning); + + case SRE_AT_BEGINNING_LINE: + return ((void*) ptr == state->beginning || + SRE_IS_LINEBREAK((int) ptr[-1])); + + case SRE_AT_END: + return (((void*) (ptr+1) == state->end && + SRE_IS_LINEBREAK((int) ptr[0])) || + ((void*) ptr == state->end)); + + case SRE_AT_END_LINE: + return ((void*) ptr == state->end || + SRE_IS_LINEBREAK((int) ptr[0])); + + case SRE_AT_END_STRING: + return ((void*) ptr == state->end); + + case SRE_AT_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; + + case SRE_AT_LOC_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_LOC_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_LOC_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_LOC_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; + +#if defined(HAVE_UNICODE) + case SRE_AT_UNI_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_UNI_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_UNI_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_UNI_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; +#endif + + } + + return 0; +} + +LOCAL(int) +SRE_CHARSET(SRE_CODE* set, SRE_CODE ch) +{ + /* check if character is a member of the given set */ + + int ok = 1; + + for (;;) { + switch (*set++) { + + case SRE_OP_FAILURE: + return !ok; + + case SRE_OP_LITERAL: + /* */ + if (ch == set[0]) + return ok; + set++; + break; + + case SRE_OP_CATEGORY: + /* */ + if (sre_category(set[0], (int) ch)) + return ok; + set += 1; + break; + + case SRE_OP_CHARSET: + if (sizeof(SRE_CODE) == 2) { + /* (16 bits per code word) */ + if (ch < 256 && (set[ch >> 4] & (1 << (ch & 15)))) + return ok; + set += 16; + } + else { + /* (32 bits per code word) */ + if (ch < 256 && (set[ch >> 5] & (1 << (ch & 31)))) + return ok; + set += 8; + } + break; + + case SRE_OP_RANGE: + /* */ + if (set[0] <= ch && ch <= set[1]) + return ok; + set += 2; + break; + + case SRE_OP_NEGATE: + ok = !ok; + break; + + case SRE_OP_BIGCHARSET: + /* <256 blockindices> */ + { + Py_ssize_t count, block; + count = *(set++); + + if (sizeof(SRE_CODE) == 2) { + block = ((unsigned char*)set)[ch >> 8]; + set += 128; + if (set[block*16 + ((ch & 255)>>4)] & (1 << (ch & 15))) + return ok; + set += count*16; + } + else { + /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids + * warnings when c's type supports only numbers < N+1 */ + if (!(ch & ~65535)) + block = ((unsigned char*)set)[ch >> 8]; + else + block = -1; + set += 64; + if (block >=0 && + (set[block*8 + ((ch & 255)>>5)] & (1 << (ch & 31)))) + return ok; + set += count*8; + } + break; + } + + default: + /* internal error -- there's not much we can do about it + here, so let's just pretend it didn't match... */ + return 0; + } + } +} + +LOCAL(Py_ssize_t) SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern); + +LOCAL(Py_ssize_t) +SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) +{ + SRE_CODE chr; + SRE_CHAR* ptr = (SRE_CHAR *)state->ptr; + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t i; + + /* adjust end */ + if (maxcount < end - ptr && maxcount != 65535) + end = ptr + maxcount; + + switch (pattern[0]) { + + case SRE_OP_IN: + /* repeated set */ + TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); + while (ptr < end && SRE_CHARSET(pattern + 2, *ptr)) + ptr++; + break; + + case SRE_OP_ANY: + /* repeated dot wildcard. */ + TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr)); + while (ptr < end && !SRE_IS_LINEBREAK(*ptr)) + ptr++; + break; + + case SRE_OP_ANY_ALL: + /* repeated dot wildcard. skip to the end of the target + string, and backtrack from there */ + TRACE(("|%p|%p|COUNT ANY_ALL\n", pattern, ptr)); + ptr = end; + break; + + case SRE_OP_LITERAL: + /* repeated literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) *ptr == chr) + ptr++; + break; + + case SRE_OP_LITERAL_IGNORE: + /* repeated literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr) + ptr++; + break; + + case SRE_OP_NOT_LITERAL: + /* repeated non-literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) *ptr != chr) + ptr++; + break; + + case SRE_OP_NOT_LITERAL_IGNORE: + /* repeated non-literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr) + ptr++; + break; + + default: + /* repeated single character pattern */ + TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); + while ((SRE_CHAR*) state->ptr < end) { + i = SRE_MATCH(state, pattern); + if (i < 0) + return i; + if (!i) + break; + } + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + (SRE_CHAR*) state->ptr - ptr)); + return (SRE_CHAR*) state->ptr - ptr; + } + + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); + return ptr - (SRE_CHAR*) state->ptr; +} + +#if 0 /* not used in this release */ +LOCAL(int) +SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) +{ + /* check if an SRE_OP_INFO block matches at the current position. + returns the number of SRE_CODE objects to skip if successful, 0 + if no match */ + + SRE_CHAR* end = state->end; + SRE_CHAR* ptr = state->ptr; + Py_ssize_t i; + + /* check minimal length */ + if (pattern[3] && (end - ptr) < pattern[3]) + return 0; + + /* check known prefix */ + if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) { + /* */ + for (i = 0; i < pattern[5]; i++) + if ((SRE_CODE) ptr[i] != pattern[7 + i]) + return 0; + return pattern[0] + 2 * pattern[6]; + } + return pattern[0]; +} +#endif + +/* The macros below should be used to protect recursive SRE_MATCH() + * calls that *failed* and do *not* return immediately (IOW, those + * that will backtrack). Explaining: + * + * - Recursive SRE_MATCH() returned true: that's usually a success + * (besides atypical cases like ASSERT_NOT), therefore there's no + * reason to restore lastmark; + * + * - Recursive SRE_MATCH() returned false but the current SRE_MATCH() + * is returning to the caller: If the current SRE_MATCH() is the + * top function of the recursion, returning false will be a matching + * failure, and it doesn't matter where lastmark is pointing to. + * If it's *not* the top function, it will be a recursive SRE_MATCH() + * failure by itself, and the calling SRE_MATCH() will have to deal + * with the failure by the same rules explained here (it will restore + * lastmark by itself if necessary); + * + * - Recursive SRE_MATCH() returned false, and will continue the + * outside 'for' loop: must be protected when breaking, since the next + * OP could potentially depend on lastmark; + * + * - Recursive SRE_MATCH() returned false, and will be called again + * inside a local for/while loop: must be protected between each + * loop iteration, since the recursive SRE_MATCH() could do anything, + * and could potentially depend on lastmark. + * + * For more information, check the discussion at SF patch #712900. + */ +#define LASTMARK_SAVE() \ + do { \ + ctx->lastmark = state->lastmark; \ + ctx->lastindex = state->lastindex; \ + } while (0) +#define LASTMARK_RESTORE() \ + do { \ + state->lastmark = ctx->lastmark; \ + state->lastindex = ctx->lastindex; \ + } while (0) + +#define RETURN_ERROR(i) do { return i; } while(0) +#define RETURN_FAILURE do { ret = 0; goto exit; } while(0) +#define RETURN_SUCCESS do { ret = 1; goto exit; } while(0) + +#define RETURN_ON_ERROR(i) \ + do { if (i < 0) RETURN_ERROR(i); } while (0) +#define RETURN_ON_SUCCESS(i) \ + do { RETURN_ON_ERROR(i); if (i > 0) RETURN_SUCCESS; } while (0) +#define RETURN_ON_FAILURE(i) \ + do { RETURN_ON_ERROR(i); if (i == 0) RETURN_FAILURE; } while (0) + +#define SFY(x) #x + +#define DATA_STACK_ALLOC(state, type, ptr) \ +do { \ + alloc_pos = state->data_stack_base; \ + TRACE(("allocating %s in %d (%d)\n", \ + SFY(type), alloc_pos, sizeof(type))); \ + if (state->data_stack_size < alloc_pos+sizeof(type)) { \ + int j = data_stack_grow(state, sizeof(type)); \ + if (j < 0) return j; \ + if (ctx_pos != -1) \ + DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ + } \ + ptr = (type*)(state->data_stack+alloc_pos); \ + state->data_stack_base += sizeof(type); \ +} while (0) + +#define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ +do { \ + TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + ptr = (type*)(state->data_stack+pos); \ +} while (0) + +#define DATA_STACK_PUSH(state, data, size) \ +do { \ + TRACE(("copy data in %p to %d (%d)\n", \ + data, state->data_stack_base, size)); \ + if (state->data_stack_size < state->data_stack_base+size) { \ + int j = data_stack_grow(state, size); \ + if (j < 0) return j; \ + if (ctx_pos != -1) \ + DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ + } \ + memcpy(state->data_stack+state->data_stack_base, data, size); \ + state->data_stack_base += size; \ +} while (0) + +#define DATA_STACK_POP(state, data, size, discard) \ +do { \ + TRACE(("copy data to %p from %d (%d)\n", \ + data, state->data_stack_base-size, size)); \ + memcpy(data, state->data_stack+state->data_stack_base-size, size); \ + if (discard) \ + state->data_stack_base -= size; \ +} while (0) + +#define DATA_STACK_POP_DISCARD(state, size) \ +do { \ + TRACE(("discard data from %d (%d)\n", \ + state->data_stack_base-size, size)); \ + state->data_stack_base -= size; \ +} while(0) + +#define DATA_PUSH(x) \ + DATA_STACK_PUSH(state, (x), sizeof(*(x))) +#define DATA_POP(x) \ + DATA_STACK_POP(state, (x), sizeof(*(x)), 1) +#define DATA_POP_DISCARD(x) \ + DATA_STACK_POP_DISCARD(state, sizeof(*(x))) +#define DATA_ALLOC(t,p) \ + DATA_STACK_ALLOC(state, t, p) +#define DATA_LOOKUP_AT(t,p,pos) \ + DATA_STACK_LOOKUP_AT(state,t,p,pos) + +#define MARK_PUSH(lastmark) \ + do if (lastmark > 0) { \ + i = lastmark; /* ctx->lastmark may change if reallocated */ \ + DATA_STACK_PUSH(state, state->mark, (i+1)*sizeof(void*)); \ + } while (0) +#define MARK_POP(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 1); \ + } while (0) +#define MARK_POP_KEEP(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 0); \ + } while (0) +#define MARK_POP_DISCARD(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP_DISCARD(state, (lastmark+1)*sizeof(void*)); \ + } while (0) + +#define JUMP_NONE 0 +#define JUMP_MAX_UNTIL_1 1 +#define JUMP_MAX_UNTIL_2 2 +#define JUMP_MAX_UNTIL_3 3 +#define JUMP_MIN_UNTIL_1 4 +#define JUMP_MIN_UNTIL_2 5 +#define JUMP_MIN_UNTIL_3 6 +#define JUMP_REPEAT 7 +#define JUMP_REPEAT_ONE_1 8 +#define JUMP_REPEAT_ONE_2 9 +#define JUMP_MIN_REPEAT_ONE 10 +#define JUMP_BRANCH 11 +#define JUMP_ASSERT 12 +#define JUMP_ASSERT_NOT 13 + +#define DO_JUMP(jumpvalue, jumplabel, nextpattern) \ + DATA_ALLOC(SRE_MATCH_CONTEXT, nextctx); \ + nextctx->last_ctx_pos = ctx_pos; \ + nextctx->jump = jumpvalue; \ + nextctx->pattern = nextpattern; \ + ctx_pos = alloc_pos; \ + ctx = nextctx; \ + goto entrance; \ + jumplabel: \ + while (0) /* gcc doesn't like labels at end of scopes */ \ + +typedef struct { + Py_ssize_t last_ctx_pos; + Py_ssize_t jump; + SRE_CHAR* ptr; + SRE_CODE* pattern; + Py_ssize_t count; + Py_ssize_t lastmark; + Py_ssize_t lastindex; + union { + SRE_CODE chr; + SRE_REPEAT* rep; + } u; +} SRE_MATCH_CONTEXT; + +/* check if string matches the given pattern. returns <0 for + error, 0 for failure, and 1 for success */ +LOCAL(Py_ssize_t) +SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) +{ + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t alloc_pos, ctx_pos = -1; + Py_ssize_t i, ret = 0; + Py_ssize_t jump; + unsigned int sigcount=0; + + SRE_MATCH_CONTEXT* ctx; + SRE_MATCH_CONTEXT* nextctx; + + TRACE(("|%p|%p|ENTER\n", pattern, state->ptr)); + + DATA_ALLOC(SRE_MATCH_CONTEXT, ctx); + ctx->last_ctx_pos = -1; + ctx->jump = JUMP_NONE; + ctx->pattern = pattern; + ctx_pos = alloc_pos; + +entrance: + + ctx->ptr = (SRE_CHAR *)state->ptr; + + if (ctx->pattern[0] == SRE_OP_INFO) { + /* optimization info block */ + /* <1=skip> <2=flags> <3=min> ... */ + if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { + TRACE(("reject (got %d chars, need %d)\n", + (end - ctx->ptr), ctx->pattern[3])); + RETURN_FAILURE; + } + ctx->pattern += ctx->pattern[1] + 1; + } + + for (;;) { + ++sigcount; + if ((0 == (sigcount & 0xfff)) && PyErr_CheckSignals()) + RETURN_ERROR(SRE_ERROR_INTERRUPTED); + + switch (*ctx->pattern++) { + + case SRE_OP_MARK: + /* set mark */ + /* */ + TRACE(("|%p|%p|MARK %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + if (i & 1) + state->lastindex = i/2 + 1; + if (i > state->lastmark) { + /* state->lastmark is the highest valid index in the + state->mark array. If it is increased by more than 1, + the intervening marks must be set to NULL to signal + that these marks have not been encountered. */ + Py_ssize_t j = state->lastmark + 1; + while (j < i) + state->mark[j++] = NULL; + state->lastmark = i; + } + state->mark[i] = ctx->ptr; + ctx->pattern++; + break; + + case SRE_OP_LITERAL: + /* match literal string */ + /* */ + TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0]) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_NOT_LITERAL: + /* match anything that is not literal character */ + /* */ + TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0]) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_SUCCESS: + /* end of pattern */ + TRACE(("|%p|%p|SUCCESS\n", ctx->pattern, ctx->ptr)); + state->ptr = ctx->ptr; + RETURN_SUCCESS; + + case SRE_OP_AT: + /* match at given position */ + /* */ + TRACE(("|%p|%p|AT %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); + if (!SRE_AT(state, ctx->ptr, *ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + break; + + case SRE_OP_CATEGORY: + /* match at given category */ + /* */ + TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0])) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_ANY: + /* match anything (except a newline) */ + /* */ + TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0])) + RETURN_FAILURE; + ctx->ptr++; + break; + + case SRE_OP_ANY_ALL: + /* match anything */ + /* */ + TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end) + RETURN_FAILURE; + ctx->ptr++; + break; + + case SRE_OP_IN: + /* match set member (or non_member) */ + /* */ + TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr)) + RETURN_FAILURE; + ctx->pattern += ctx->pattern[0]; + ctx->ptr++; + break; + + case SRE_OP_LITERAL_IGNORE: + TRACE(("|%p|%p|LITERAL_IGNORE %d\n", + ctx->pattern, ctx->ptr, ctx->pattern[0])); + if (ctx->ptr >= end || + state->lower(*ctx->ptr) != state->lower(*ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_NOT_LITERAL_IGNORE: + TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", + ctx->pattern, ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || + state->lower(*ctx->ptr) == state->lower(*ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_IN_IGNORE: + TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end + || !SRE_CHARSET(ctx->pattern+1, + (SRE_CODE)state->lower(*ctx->ptr))) + RETURN_FAILURE; + ctx->pattern += ctx->pattern[0]; + ctx->ptr++; + break; + + case SRE_OP_JUMP: + case SRE_OP_INFO: + /* jump forward */ + /* */ + TRACE(("|%p|%p|JUMP %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_BRANCH: + /* alternation */ + /* <0=skip> code ... */ + TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); + LASTMARK_SAVE(); + ctx->u.rep = state->repeat; + if (ctx->u.rep) + MARK_PUSH(ctx->lastmark); + for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { + if (ctx->pattern[1] == SRE_OP_LITERAL && + (ctx->ptr >= end || + (SRE_CODE) *ctx->ptr != ctx->pattern[2])) + continue; + if (ctx->pattern[1] == SRE_OP_IN && + (ctx->ptr >= end || + !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) + continue; + state->ptr = ctx->ptr; + DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); + if (ret) { + if (ctx->u.rep) + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + if (ctx->u.rep) + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + } + if (ctx->u.rep) + MARK_POP_DISCARD(ctx->lastmark); + RETURN_FAILURE; + + case SRE_OP_REPEAT_ONE: + /* match repeated sequence (maximizing regexp) */ + + /* this operator only works if the repeated item is + exactly one character wide, and we're not already + collecting backtracking points. for other cases, + use the MAX_REPEAT operator */ + + /* <1=min> <2=max> item tail */ + + TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + if (ctx->ptr + ctx->pattern[1] > end) + RETURN_FAILURE; /* cannot match */ + + state->ptr = ctx->ptr; + + ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[2]); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + ctx->count = ret; + ctx->ptr += ctx->count; + + /* when we arrive here, count contains the number of + matches, and ctx->ptr points to the tail of the target + string. check if the rest of the pattern matches, + and backtrack if not. */ + + if (ctx->count < (Py_ssize_t) ctx->pattern[1]) + RETURN_FAILURE; + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { + /* tail is empty. we're finished */ + state->ptr = ctx->ptr; + RETURN_SUCCESS; + } + + LASTMARK_SAVE(); + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { + /* tail starts with a literal. skip positions where + the rest of the pattern cannot possibly match */ + ctx->u.chr = ctx->pattern[ctx->pattern[0]+1]; + for (;;) { + while (ctx->count >= (Py_ssize_t) ctx->pattern[1] && + (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) { + ctx->ptr--; + ctx->count--; + } + if (ctx->count < (Py_ssize_t) ctx->pattern[1]) + break; + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + + LASTMARK_RESTORE(); + + ctx->ptr--; + ctx->count--; + } + + } else { + /* general case */ + while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->ptr--; + ctx->count--; + LASTMARK_RESTORE(); + } + } + RETURN_FAILURE; + + case SRE_OP_MIN_REPEAT_ONE: + /* match repeated sequence (minimizing regexp) */ + + /* this operator only works if the repeated item is + exactly one character wide, and we're not already + collecting backtracking points. for other cases, + use the MIN_REPEAT operator */ + + /* <1=min> <2=max> item tail */ + + TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + if (ctx->ptr + ctx->pattern[1] > end) + RETURN_FAILURE; /* cannot match */ + + state->ptr = ctx->ptr; + + if (ctx->pattern[1] == 0) + ctx->count = 0; + else { + /* count using pattern min as the maximum */ + ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[1]); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + if (ret < (Py_ssize_t) ctx->pattern[1]) + /* didn't match minimum number of times */ + RETURN_FAILURE; + /* advance past minimum matches of repeat */ + ctx->count = ret; + ctx->ptr += ctx->count; + } + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { + /* tail is empty. we're finished */ + state->ptr = ctx->ptr; + RETURN_SUCCESS; + + } else { + /* general case */ + LASTMARK_SAVE(); + while ((Py_ssize_t)ctx->pattern[2] == 65535 + || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { + state->ptr = ctx->ptr; + DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + state->ptr = ctx->ptr; + ret = SRE_COUNT(state, ctx->pattern+3, 1); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + if (ret == 0) + break; + assert(ret == 1); + ctx->ptr++; + ctx->count++; + LASTMARK_RESTORE(); + } + } + RETURN_FAILURE; + + case SRE_OP_REPEAT: + /* create repeat context. all the hard work is done + by the UNTIL operator (MAX_UNTIL, MIN_UNTIL) */ + /* <1=min> <2=max> item tail */ + TRACE(("|%p|%p|REPEAT %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + /* install new repeat context */ + ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep)); + if (!ctx->u.rep) { + PyErr_NoMemory(); + RETURN_FAILURE; + } + ctx->u.rep->count = -1; + ctx->u.rep->pattern = ctx->pattern; + ctx->u.rep->prev = state->repeat; + ctx->u.rep->last_ptr = NULL; + state->repeat = ctx->u.rep; + + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT, jump_repeat, ctx->pattern+ctx->pattern[0]); + state->repeat = ctx->u.rep->prev; + PyObject_FREE(ctx->u.rep); + + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + RETURN_FAILURE; + + case SRE_OP_MAX_UNTIL: + /* maximizing repeat */ + /* <1=min> <2=max> item tail */ + + /* FIXME: we probably need to deal with zero-width + matches in here... */ + + ctx->u.rep = state->repeat; + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_STATE); + + state->ptr = ctx->ptr; + + ctx->count = ctx->u.rep->count+1; + + TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + ctx->ptr, ctx->count)); + + if (ctx->count < ctx->u.rep->pattern[1]) { + /* not enough matches */ + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + } + + if ((ctx->count < ctx->u.rep->pattern[2] || + ctx->u.rep->pattern[2] == 65535) && + state->ptr != ctx->u.rep->last_ptr) { + /* we may have enough matches, but if we can + match another item, do so */ + ctx->u.rep->count = ctx->count; + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + /* zero-width match protection */ + DATA_PUSH(&ctx->u.rep->last_ptr); + ctx->u.rep->last_ptr = state->ptr; + DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, + ctx->u.rep->pattern+3); + DATA_POP(&ctx->u.rep->last_ptr); + if (ret) { + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + } + + /* cannot match more repeated items here. make sure the + tail matches */ + state->repeat = ctx->u.rep->prev; + DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern); + RETURN_ON_SUCCESS(ret); + state->repeat = ctx->u.rep; + state->ptr = ctx->ptr; + RETURN_FAILURE; + + case SRE_OP_MIN_UNTIL: + /* minimizing repeat */ + /* <1=min> <2=max> item tail */ + + ctx->u.rep = state->repeat; + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_STATE); + + state->ptr = ctx->ptr; + + ctx->count = ctx->u.rep->count+1; + + TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + ctx->ptr, ctx->count, ctx->u.rep->pattern)); + + if (ctx->count < ctx->u.rep->pattern[1]) { + /* not enough matches */ + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + } + + LASTMARK_SAVE(); + + /* see if the tail matches */ + state->repeat = ctx->u.rep->prev; + DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + + state->repeat = ctx->u.rep; + state->ptr = ctx->ptr; + + LASTMARK_RESTORE(); + + if (ctx->count >= ctx->u.rep->pattern[2] + && ctx->u.rep->pattern[2] != 65535) + RETURN_FAILURE; + + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + + case SRE_OP_GROUPREF: + /* match backreference */ + TRACE(("|%p|%p|GROUPREF %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + RETURN_FAILURE; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) + RETURN_FAILURE; + while (p < e) { + if (ctx->ptr >= end || *ctx->ptr != *p) + RETURN_FAILURE; + p++; ctx->ptr++; + } + } + } + ctx->pattern++; + break; + + case SRE_OP_GROUPREF_IGNORE: + /* match backreference */ + TRACE(("|%p|%p|GROUPREF_IGNORE %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + RETURN_FAILURE; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) + RETURN_FAILURE; + while (p < e) { + if (ctx->ptr >= end || + state->lower(*ctx->ptr) != state->lower(*p)) + RETURN_FAILURE; + p++; ctx->ptr++; + } + } + } + ctx->pattern++; + break; + + case SRE_OP_GROUPREF_EXISTS: + TRACE(("|%p|%p|GROUPREF_EXISTS %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + /* codeyes codeno ... */ + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + ctx->pattern += ctx->pattern[1]; + break; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) { + ctx->pattern += ctx->pattern[1]; + break; + } + } + } + ctx->pattern += 2; + break; + + case SRE_OP_ASSERT: + /* assert subpattern */ + /* */ + TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[1])); + state->ptr = ctx->ptr - ctx->pattern[1]; + if (state->ptr < state->beginning) + RETURN_FAILURE; + DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2); + RETURN_ON_FAILURE(ret); + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_ASSERT_NOT: + /* assert not subpattern */ + /* */ + TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[1])); + state->ptr = ctx->ptr - ctx->pattern[1]; + if (state->ptr >= state->beginning) { + DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_FAILURE; + } + } + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_FAILURE: + /* immediate failure */ + TRACE(("|%p|%p|FAILURE\n", ctx->pattern, ctx->ptr)); + RETURN_FAILURE; + + default: + TRACE(("|%p|%p|UNKNOWN %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[-1])); + RETURN_ERROR(SRE_ERROR_ILLEGAL); + } + } + +exit: + ctx_pos = ctx->last_ctx_pos; + jump = ctx->jump; + DATA_POP_DISCARD(ctx); + if (ctx_pos == -1) + return ret; + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + + switch (jump) { + case JUMP_MAX_UNTIL_2: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_2\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_2; + case JUMP_MAX_UNTIL_3: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_3\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_3; + case JUMP_MIN_UNTIL_2: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_2\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_2; + case JUMP_MIN_UNTIL_3: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_3\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_3; + case JUMP_BRANCH: + TRACE(("|%p|%p|JUMP_BRANCH\n", ctx->pattern, ctx->ptr)); + goto jump_branch; + case JUMP_MAX_UNTIL_1: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_1\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_1; + case JUMP_MIN_UNTIL_1: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_1\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_1; + case JUMP_REPEAT: + TRACE(("|%p|%p|JUMP_REPEAT\n", ctx->pattern, ctx->ptr)); + goto jump_repeat; + case JUMP_REPEAT_ONE_1: + TRACE(("|%p|%p|JUMP_REPEAT_ONE_1\n", ctx->pattern, ctx->ptr)); + goto jump_repeat_one_1; + case JUMP_REPEAT_ONE_2: + TRACE(("|%p|%p|JUMP_REPEAT_ONE_2\n", ctx->pattern, ctx->ptr)); + goto jump_repeat_one_2; + case JUMP_MIN_REPEAT_ONE: + TRACE(("|%p|%p|JUMP_MIN_REPEAT_ONE\n", ctx->pattern, ctx->ptr)); + goto jump_min_repeat_one; + case JUMP_ASSERT: + TRACE(("|%p|%p|JUMP_ASSERT\n", ctx->pattern, ctx->ptr)); + goto jump_assert; + case JUMP_ASSERT_NOT: + TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); + goto jump_assert_not; + case JUMP_NONE: + TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + break; + } + + return ret; /* should never get here */ +} + +LOCAL(Py_ssize_t) +SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) +{ + SRE_CHAR* ptr = (SRE_CHAR *)state->start; + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t status = 0; + Py_ssize_t prefix_len = 0; + Py_ssize_t prefix_skip = 0; + SRE_CODE* prefix = NULL; + SRE_CODE* charset = NULL; + SRE_CODE* overlap = NULL; + int flags = 0; + + if (pattern[0] == SRE_OP_INFO) { + /* optimization info block */ + /* <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> */ + + flags = pattern[2]; + + if (pattern[3] > 1) { + /* adjust end point (but make sure we leave at least one + character in there, so literal search will work) */ + end -= pattern[3]-1; + if (end <= ptr) + end = ptr+1; + } + + if (flags & SRE_INFO_PREFIX) { + /* pattern starts with a known prefix */ + /* */ + prefix_len = pattern[5]; + prefix_skip = pattern[6]; + prefix = pattern + 7; + overlap = prefix + prefix_len - 1; + } else if (flags & SRE_INFO_CHARSET) + /* pattern starts with a character from a known set */ + /* */ + charset = pattern + 5; + + pattern += 1 + pattern[1]; + } + + TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("charset = %p\n", charset)); + +#if defined(USE_FAST_SEARCH) + if (prefix_len > 1) { + /* pattern starts with a known prefix. use the overlap + table to skip forward as fast as we possibly can */ + Py_ssize_t i = 0; + end = (SRE_CHAR *)state->end; + while (ptr < end) { + for (;;) { + if ((SRE_CODE) ptr[0] != prefix[i]) { + if (!i) + break; + else + i = overlap[i]; + } else { + if (++i == prefix_len) { + /* found a potential match */ + TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr)); + state->start = ptr + 1 - prefix_len; + state->ptr = ptr + 1 - prefix_len + prefix_skip; + if (flags & SRE_INFO_LITERAL) + return 1; /* we got all of it */ + status = SRE_MATCH(state, pattern + 2*prefix_skip); + if (status != 0) + return status; + /* close but no cigar -- try again */ + i = overlap[i]; + } + break; + } + } + ptr++; + } + return 0; + } +#endif + + if (pattern[0] == SRE_OP_LITERAL) { + /* pattern starts with a literal character. this is used + for short prefixes, and if fast search is disabled */ + SRE_CODE chr = pattern[1]; + end = (SRE_CHAR *)state->end; + for (;;) { + while (ptr < end && (SRE_CODE) ptr[0] != chr) + ptr++; + if (ptr >= end) + return 0; + TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); + state->start = ptr; + state->ptr = ++ptr; + if (flags & SRE_INFO_LITERAL) + return 1; /* we got all of it */ + status = SRE_MATCH(state, pattern + 2); + if (status != 0) + break; + } + } else if (charset) { + /* pattern starts with a character from a known set */ + end = (SRE_CHAR *)state->end; + for (;;) { + while (ptr < end && !SRE_CHARSET(charset, ptr[0])) + ptr++; + if (ptr >= end) + return 0; + TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr)); + state->start = ptr; + state->ptr = ptr; + status = SRE_MATCH(state, pattern); + if (status != 0) + break; + ptr++; + } + } else + /* general case */ + while (ptr <= end) { + TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); + state->start = state->ptr = ptr++; + status = SRE_MATCH(state, pattern); + if (status != 0) + break; + } + + return status; +} + +LOCAL(int) +SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len) +{ + /* check if given string is a literal template (i.e. no escapes) */ + while (len-- > 0) + if (*ptr++ == '\\') + return 0; + return 1; +} + +#if !defined(SRE_RECURSIVE) + +/* -------------------------------------------------------------------- */ +/* factories and destructors */ + +/* see sre.h for object declarations */ +static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int); +static PyObject*pattern_scanner(PatternObject*, PyObject*); + +static PyObject * +sre_codesize(PyObject* self, PyObject *unused) +{ + return Py_BuildValue("l", sizeof(SRE_CODE)); +} + +static PyObject * +sre_getlower(PyObject* self, PyObject* args) +{ + int character, flags; + if (!PyArg_ParseTuple(args, "ii", &character, &flags)) + return NULL; + if (flags & SRE_FLAG_LOCALE) + return Py_BuildValue("i", sre_lower_locale(character)); + if (flags & SRE_FLAG_UNICODE) +#if defined(HAVE_UNICODE) + return Py_BuildValue("i", sre_lower_unicode(character)); +#else + return Py_BuildValue("i", sre_lower_locale(character)); +#endif + return Py_BuildValue("i", sre_lower(character)); +} + +LOCAL(void) +state_reset(SRE_STATE* state) +{ + /* FIXME: dynamic! */ + /*memset(state->mark, 0, sizeof(*state->mark) * SRE_MARK_SIZE);*/ + + state->lastmark = -1; + state->lastindex = -1; + + state->repeat = NULL; + + data_stack_dealloc(state); +} + +static void* +getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) +{ + /* given a python object, return a data pointer, a length (in + characters), and a character size. return NULL if the object + is not a string (or not compatible) */ + + PyBufferProcs *buffer; + Py_ssize_t size, bytes; + int charsize; + void* ptr; + +#if defined(HAVE_UNICODE) + if (PyUnicode_Check(string)) { + /* unicode strings doesn't always support the buffer interface */ + ptr = (void*) PyUnicode_AS_DATA(string); + bytes = PyUnicode_GET_DATA_SIZE(string); + size = PyUnicode_GET_SIZE(string); + charsize = sizeof(Py_UNICODE); + + } else { +#endif + + /* get pointer to string buffer */ + buffer = Py_TYPE(string)->tp_as_buffer; + if (!buffer || !buffer->bf_getreadbuffer || !buffer->bf_getsegcount || + buffer->bf_getsegcount(string, NULL) != 1) { + PyErr_SetString(PyExc_TypeError, "expected string or buffer"); + return NULL; + } + + /* determine buffer size */ + bytes = buffer->bf_getreadbuffer(string, 0, &ptr); + if (bytes < 0) { + PyErr_SetString(PyExc_TypeError, "buffer has negative size"); + return NULL; + } + + /* determine character size */ +#if PY_VERSION_HEX >= 0x01060000 + size = PyObject_Size(string); +#else + size = PyObject_Length(string); +#endif + + if (PyString_Check(string) || bytes == size) + charsize = 1; +#if defined(HAVE_UNICODE) + else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) + charsize = sizeof(Py_UNICODE); +#endif + else { + PyErr_SetString(PyExc_TypeError, "buffer size mismatch"); + return NULL; + } + +#if defined(HAVE_UNICODE) + } +#endif + + *p_length = size; + *p_charsize = charsize; + + return ptr; +} + +LOCAL(PyObject*) +state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, + Py_ssize_t start, Py_ssize_t end) +{ + /* prepare state object */ + + Py_ssize_t length; + int charsize; + void* ptr; + + memset(state, 0, sizeof(SRE_STATE)); + + state->lastmark = -1; + state->lastindex = -1; + + ptr = getstring(string, &length, &charsize); + if (!ptr) + return NULL; + + /* adjust boundaries */ + if (start < 0) + start = 0; + else if (start > length) + start = length; + + if (end < 0) + end = 0; + else if (end > length) + end = length; + + state->charsize = charsize; + + state->beginning = ptr; + + state->start = (void*) ((char*) ptr + start * state->charsize); + state->end = (void*) ((char*) ptr + end * state->charsize); + + Py_INCREF(string); + state->string = string; + state->pos = start; + state->endpos = end; + + if (pattern->flags & SRE_FLAG_LOCALE) + state->lower = sre_lower_locale; + else if (pattern->flags & SRE_FLAG_UNICODE) +#if defined(HAVE_UNICODE) + state->lower = sre_lower_unicode; +#else + state->lower = sre_lower_locale; +#endif + else + state->lower = sre_lower; + + return string; +} + +LOCAL(void) +state_fini(SRE_STATE* state) +{ + Py_XDECREF(state->string); + data_stack_dealloc(state); +} + +/* calculate offset from start of string */ +#define STATE_OFFSET(state, member)\ + (((char*)(member) - (char*)(state)->beginning) / (state)->charsize) + +LOCAL(PyObject*) +state_getslice(SRE_STATE* state, Py_ssize_t index, PyObject* string, int empty) +{ + Py_ssize_t i, j; + + index = (index - 1) * 2; + + if (string == Py_None || index >= state->lastmark || !state->mark[index] || !state->mark[index+1]) { + if (empty) + /* want empty string */ + i = j = 0; + else { + Py_INCREF(Py_None); + return Py_None; + } + } else { + i = STATE_OFFSET(state, state->mark[index]); + j = STATE_OFFSET(state, state->mark[index+1]); + } + + return PySequence_GetSlice(string, i, j); +} + +static void +pattern_error(int status) +{ + switch (status) { + case SRE_ERROR_RECURSION_LIMIT: + PyErr_SetString( + PyExc_RuntimeError, + "maximum recursion limit exceeded" + ); + break; + case SRE_ERROR_MEMORY: + PyErr_NoMemory(); + break; + case SRE_ERROR_INTERRUPTED: + /* An exception has already been raised, so let it fly */ + break; + default: + /* other error codes indicate compiler/engine bugs */ + PyErr_SetString( + PyExc_RuntimeError, + "internal error in regular expression engine" + ); + } +} + +static void +pattern_dealloc(PatternObject* self) +{ + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_XDECREF(self->pattern); + Py_XDECREF(self->groupindex); + Py_XDECREF(self->indexgroup); + PyObject_DEL(self); +} + +static PyObject* +pattern_match(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + int status; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + static char* kwlist[] = { "pattern", "pos", "endpos", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:match", kwlist, + &string, &start, &end)) + return NULL; + + string = state_init(&state, self, string, start, end); + if (!string) + return NULL; + + state.ptr = state.start; + + TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); + + if (state.charsize == 1) { + status = sre_match(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_umatch(&state, PatternObject_GetCode(self)); +#endif + } + + TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); + if (PyErr_Occurred()) + return NULL; + + state_fini(&state); + + return pattern_new_match(self, &state, status); +} + +static PyObject* +pattern_search(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + int status; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + static char* kwlist[] = { "pattern", "pos", "endpos", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:search", kwlist, + &string, &start, &end)) + return NULL; + + string = state_init(&state, self, string, start, end); + if (!string) + return NULL; + + TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr)); + + if (state.charsize == 1) { + status = sre_search(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(&state, PatternObject_GetCode(self)); +#endif + } + + TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); + + state_fini(&state); + + if (PyErr_Occurred()) + return NULL; + + return pattern_new_match(self, &state, status); +} + +static PyObject* +call(char* module, char* function, PyObject* args) +{ + PyObject* name; + PyObject* mod; + PyObject* func; + PyObject* result; + + if (!args) + return NULL; + name = PyString_FromString(module); + if (!name) + return NULL; + mod = PyImport_Import(name); + Py_DECREF(name); + if (!mod) + return NULL; + func = PyObject_GetAttrString(mod, function); + Py_DECREF(mod); + if (!func) + return NULL; + result = PyObject_CallObject(func, args); + Py_DECREF(func); + Py_DECREF(args); + return result; +} + +#ifdef USE_BUILTIN_COPY +static int +deepcopy(PyObject** object, PyObject* memo) +{ + PyObject* copy; + + copy = call( + "copy", "deepcopy", + PyTuple_Pack(2, *object, memo) + ); + if (!copy) + return 0; + + Py_DECREF(*object); + *object = copy; + + return 1; /* success */ +} +#endif + +static PyObject* +join_list(PyObject* list, PyObject* string) +{ + /* join list elements */ + + PyObject* joiner; +#if PY_VERSION_HEX >= 0x01060000 + PyObject* function; + PyObject* args; +#endif + PyObject* result; + + joiner = PySequence_GetSlice(string, 0, 0); + if (!joiner) + return NULL; + + if (PyList_GET_SIZE(list) == 0) { + Py_DECREF(list); + return joiner; + } + +#if PY_VERSION_HEX >= 0x01060000 + function = PyObject_GetAttrString(joiner, "join"); + if (!function) { + Py_DECREF(joiner); + return NULL; + } + args = PyTuple_New(1); + if (!args) { + Py_DECREF(function); + Py_DECREF(joiner); + return NULL; + } + PyTuple_SET_ITEM(args, 0, list); + result = PyObject_CallObject(function, args); + Py_DECREF(args); /* also removes list */ + Py_DECREF(function); +#else + result = call( + "string", "join", + PyTuple_Pack(2, list, joiner) + ); +#endif + Py_DECREF(joiner); + + return result; +} + +static PyObject* +pattern_findall(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + PyObject* list; + int status; + Py_ssize_t i, b, e; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + static char* kwlist[] = { "source", "pos", "endpos", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:findall", kwlist, + &string, &start, &end)) + return NULL; + + string = state_init(&state, self, string, start, end); + if (!string) + return NULL; + + list = PyList_New(0); + if (!list) { + state_fini(&state); + return NULL; + } + + while (state.start <= state.end) { + + PyObject* item; + + state_reset(&state); + + state.ptr = state.start; + + if (state.charsize == 1) { + status = sre_search(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(&state, PatternObject_GetCode(self)); +#endif + } + + if (PyErr_Occurred()) + goto error; + + if (status <= 0) { + if (status == 0) + break; + pattern_error(status); + goto error; + } + + /* don't bother to build a match object */ + switch (self->groups) { + case 0: + b = STATE_OFFSET(&state, state.start); + e = STATE_OFFSET(&state, state.ptr); + item = PySequence_GetSlice(string, b, e); + if (!item) + goto error; + break; + case 1: + item = state_getslice(&state, 1, string, 1); + if (!item) + goto error; + break; + default: + item = PyTuple_New(self->groups); + if (!item) + goto error; + for (i = 0; i < self->groups; i++) { + PyObject* o = state_getslice(&state, i+1, string, 1); + if (!o) { + Py_DECREF(item); + goto error; + } + PyTuple_SET_ITEM(item, i, o); + } + break; + } + + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + + if (state.ptr == state.start) + state.start = (void*) ((char*) state.ptr + state.charsize); + else + state.start = state.ptr; + + } + + state_fini(&state); + return list; + +error: + Py_DECREF(list); + state_fini(&state); + return NULL; + +} + +#if PY_VERSION_HEX >= 0x02020000 +static PyObject* +pattern_finditer(PatternObject* pattern, PyObject* args) +{ + PyObject* scanner; + PyObject* search; + PyObject* iterator; + + scanner = pattern_scanner(pattern, args); + if (!scanner) + return NULL; + + search = PyObject_GetAttrString(scanner, "search"); + Py_DECREF(scanner); + if (!search) + return NULL; + + iterator = PyCallIter_New(search, Py_None); + Py_DECREF(search); + + return iterator; +} +#endif + +static PyObject* +pattern_split(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + PyObject* list; + PyObject* item; + int status; + Py_ssize_t n; + Py_ssize_t i; + void* last; + + PyObject* string; + Py_ssize_t maxsplit = 0; + static char* kwlist[] = { "source", "maxsplit", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|n:split", kwlist, + &string, &maxsplit)) + return NULL; + + string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX); + if (!string) + return NULL; + + list = PyList_New(0); + if (!list) { + state_fini(&state); + return NULL; + } + + n = 0; + last = state.start; + + while (!maxsplit || n < maxsplit) { + + state_reset(&state); + + state.ptr = state.start; + + if (state.charsize == 1) { + status = sre_search(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(&state, PatternObject_GetCode(self)); +#endif + } + + if (PyErr_Occurred()) + goto error; + + if (status <= 0) { + if (status == 0) + break; + pattern_error(status); + goto error; + } + + if (state.start == state.ptr) { + if (last == state.end) + break; + /* skip one character */ + state.start = (void*) ((char*) state.ptr + state.charsize); + continue; + } + + /* get segment before this match */ + item = PySequence_GetSlice( + string, STATE_OFFSET(&state, last), + STATE_OFFSET(&state, state.start) + ); + if (!item) + goto error; + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + + /* add groups (if any) */ + for (i = 0; i < self->groups; i++) { + item = state_getslice(&state, i+1, string, 0); + if (!item) + goto error; + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + } + + n = n + 1; + + last = state.start = state.ptr; + + } + + /* get segment following last match (even if empty) */ + item = PySequence_GetSlice( + string, STATE_OFFSET(&state, last), state.endpos + ); + if (!item) + goto error; + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + + state_fini(&state); + return list; + +error: + Py_DECREF(list); + state_fini(&state); + return NULL; + +} + +static PyObject* +pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, + Py_ssize_t count, Py_ssize_t subn) +{ + SRE_STATE state; + PyObject* list; + PyObject* item; + PyObject* filter; + PyObject* args; + PyObject* match; + void* ptr; + int status; + Py_ssize_t n; + Py_ssize_t i, b, e; + int bint; + int filter_is_callable; + + if (PyCallable_Check(ptemplate)) { + /* sub/subn takes either a function or a template */ + filter = ptemplate; + Py_INCREF(filter); + filter_is_callable = 1; + } else { + /* if not callable, check if it's a literal string */ + int literal; + ptr = getstring(ptemplate, &n, &bint); + b = bint; + if (ptr) { + if (b == 1) { + literal = sre_literal_template((unsigned char *)ptr, n); + } else { +#if defined(HAVE_UNICODE) + literal = sre_uliteral_template((Py_UNICODE *)ptr, n); +#endif + } + } else { + PyErr_Clear(); + literal = 0; + } + if (literal) { + filter = ptemplate; + Py_INCREF(filter); + filter_is_callable = 0; + } else { + /* not a literal; hand it over to the template compiler */ + filter = call( + SRE_PY_MODULE, "_subx", + PyTuple_Pack(2, self, ptemplate) + ); + if (!filter) + return NULL; + filter_is_callable = PyCallable_Check(filter); + } + } + + string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX); + if (!string) { + Py_DECREF(filter); + return NULL; + } + + list = PyList_New(0); + if (!list) { + Py_DECREF(filter); + state_fini(&state); + return NULL; + } + + n = i = 0; + + while (!count || n < count) { + + state_reset(&state); + + state.ptr = state.start; + + if (state.charsize == 1) { + status = sre_search(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(&state, PatternObject_GetCode(self)); +#endif + } + + if (PyErr_Occurred()) + goto error; + + if (status <= 0) { + if (status == 0) + break; + pattern_error(status); + goto error; + } + + b = STATE_OFFSET(&state, state.start); + e = STATE_OFFSET(&state, state.ptr); + + if (i < b) { + /* get segment before this match */ + item = PySequence_GetSlice(string, i, b); + if (!item) + goto error; + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + + } else if (i == b && i == e && n > 0) + /* ignore empty match on latest position */ + goto next; + + if (filter_is_callable) { + /* pass match object through filter */ + match = pattern_new_match(self, &state, 1); + if (!match) + goto error; + args = PyTuple_Pack(1, match); + if (!args) { + Py_DECREF(match); + goto error; + } + item = PyObject_CallObject(filter, args); + Py_DECREF(args); + Py_DECREF(match); + if (!item) + goto error; + } else { + /* filter is literal string */ + item = filter; + Py_INCREF(item); + } + + /* add to list */ + if (item != Py_None) { + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + } + + i = e; + n = n + 1; + +next: + /* move on */ + if (state.ptr == state.start) + state.start = (void*) ((char*) state.ptr + state.charsize); + else + state.start = state.ptr; + + } + + /* get segment following last match */ + if (i < state.endpos) { + item = PySequence_GetSlice(string, i, state.endpos); + if (!item) + goto error; + status = PyList_Append(list, item); + Py_DECREF(item); + if (status < 0) + goto error; + } + + state_fini(&state); + + Py_DECREF(filter); + + /* convert list to single string (also removes list) */ + item = join_list(list, string); + + if (!item) + return NULL; + + if (subn) + return Py_BuildValue("Ni", item, n); + + return item; + +error: + Py_DECREF(list); + state_fini(&state); + Py_DECREF(filter); + return NULL; + +} + +static PyObject* +pattern_sub(PatternObject* self, PyObject* args, PyObject* kw) +{ + PyObject* ptemplate; + PyObject* string; + Py_ssize_t count = 0; + static char* kwlist[] = { "repl", "string", "count", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|n:sub", kwlist, + &ptemplate, &string, &count)) + return NULL; + + return pattern_subx(self, ptemplate, string, count, 0); +} + +static PyObject* +pattern_subn(PatternObject* self, PyObject* args, PyObject* kw) +{ + PyObject* ptemplate; + PyObject* string; + Py_ssize_t count = 0; + static char* kwlist[] = { "repl", "string", "count", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|n:subn", kwlist, + &ptemplate, &string, &count)) + return NULL; + + return pattern_subx(self, ptemplate, string, count, 1); +} + +static PyObject* +pattern_copy(PatternObject* self, PyObject *unused) +{ +#ifdef USE_BUILTIN_COPY + PatternObject* copy; + int offset; + + copy = PyObject_NEW_VAR(PatternObject, &Pattern_Type, self->codesize); + if (!copy) + return NULL; + + offset = offsetof(PatternObject, groups); + + Py_XINCREF(self->groupindex); + Py_XINCREF(self->indexgroup); + Py_XINCREF(self->pattern); + + memcpy((char*) copy + offset, (char*) self + offset, + sizeof(PatternObject) + self->codesize * sizeof(SRE_CODE) - offset); + copy->weakreflist = NULL; + + return (PyObject*) copy; +#else + PyErr_SetString(PyExc_TypeError, "cannot copy this pattern object"); + return NULL; +#endif +} + +static PyObject* +pattern_deepcopy(PatternObject* self, PyObject* memo) +{ +#ifdef USE_BUILTIN_COPY + PatternObject* copy; + + copy = (PatternObject*) pattern_copy(self); + if (!copy) + return NULL; + + if (!deepcopy(©->groupindex, memo) || + !deepcopy(©->indexgroup, memo) || + !deepcopy(©->pattern, memo)) { + Py_DECREF(copy); + return NULL; + } + +#else + PyErr_SetString(PyExc_TypeError, "cannot deepcopy this pattern object"); + return NULL; +#endif +} + +PyDoc_STRVAR(pattern_match_doc, +"match(string[, pos[, endpos]]) --> match object or None.\n\ + Matches zero or more characters at the beginning of the string"); + +PyDoc_STRVAR(pattern_search_doc, +"search(string[, pos[, endpos]]) --> match object or None.\n\ + Scan through string looking for a match, and return a corresponding\n\ + MatchObject instance. Return None if no position in the string matches."); + +PyDoc_STRVAR(pattern_split_doc, +"split(string[, maxsplit = 0]) --> list.\n\ + Split string by the occurrences of pattern."); + +PyDoc_STRVAR(pattern_findall_doc, +"findall(string[, pos[, endpos]]) --> list.\n\ + Return a list of all non-overlapping matches of pattern in string."); + +PyDoc_STRVAR(pattern_finditer_doc, +"finditer(string[, pos[, endpos]]) --> iterator.\n\ + Return an iterator over all non-overlapping matches for the \n\ + RE pattern in string. For each match, the iterator returns a\n\ + match object."); + +PyDoc_STRVAR(pattern_sub_doc, +"sub(repl, string[, count = 0]) --> newstring\n\ + Return the string obtained by replacing the leftmost non-overlapping\n\ + occurrences of pattern in string by the replacement repl."); + +PyDoc_STRVAR(pattern_subn_doc, +"subn(repl, string[, count = 0]) --> (newstring, number of subs)\n\ + Return the tuple (new_string, number_of_subs_made) found by replacing\n\ + the leftmost non-overlapping occurrences of pattern with the\n\ + replacement repl."); + +PyDoc_STRVAR(pattern_doc, "Compiled regular expression objects"); + +static PyMethodDef pattern_methods[] = { + {"match", (PyCFunction) pattern_match, METH_VARARGS|METH_KEYWORDS, + pattern_match_doc}, + {"search", (PyCFunction) pattern_search, METH_VARARGS|METH_KEYWORDS, + pattern_search_doc}, + {"sub", (PyCFunction) pattern_sub, METH_VARARGS|METH_KEYWORDS, + pattern_sub_doc}, + {"subn", (PyCFunction) pattern_subn, METH_VARARGS|METH_KEYWORDS, + pattern_subn_doc}, + {"split", (PyCFunction) pattern_split, METH_VARARGS|METH_KEYWORDS, + pattern_split_doc}, + {"findall", (PyCFunction) pattern_findall, METH_VARARGS|METH_KEYWORDS, + pattern_findall_doc}, +#if PY_VERSION_HEX >= 0x02020000 + {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS, + pattern_finditer_doc}, +#endif + {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS}, + {"__copy__", (PyCFunction) pattern_copy, METH_NOARGS}, + {"__deepcopy__", (PyCFunction) pattern_deepcopy, METH_O}, + {NULL, NULL} +}; + +static PyObject* +pattern_getattr(PatternObject* self, char* name) +{ + PyObject* res; + + res = Py_FindMethod(pattern_methods, (PyObject*) self, name); + + if (res) + return res; + + PyErr_Clear(); + + /* attributes */ + if (!strcmp(name, "pattern")) { + Py_INCREF(self->pattern); + return self->pattern; + } + + if (!strcmp(name, "flags")) + return Py_BuildValue("i", self->flags); + + if (!strcmp(name, "groups")) + return Py_BuildValue("i", self->groups); + + if (!strcmp(name, "groupindex") && self->groupindex) { + Py_INCREF(self->groupindex); + return self->groupindex; + } + + PyErr_SetString(PyExc_AttributeError, name); + return NULL; +} + +statichere PyTypeObject Pattern_Type = { + PyObject_HEAD_INIT(NULL) + 0, "_" SRE_MODULE ".SRE_Pattern", + sizeof(PatternObject), sizeof(SRE_CODE), + (destructor)pattern_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)pattern_getattr, /*tp_getattr*/ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */ + pattern_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PatternObject, weakreflist), /* tp_weaklistoffset */ +}; + +static int _validate(PatternObject *self); /* Forward */ + +static PyObject * +_compile(PyObject* self_, PyObject* args) +{ + /* "compile" pattern descriptor to pattern object */ + + PatternObject* self; + Py_ssize_t i, n; + + PyObject* pattern; + int flags = 0; + PyObject* code; + Py_ssize_t groups = 0; + PyObject* groupindex = NULL; + PyObject* indexgroup = NULL; + if (!PyArg_ParseTuple(args, "OiO!|nOO", &pattern, &flags, + &PyList_Type, &code, &groups, + &groupindex, &indexgroup)) + return NULL; + + n = PyList_GET_SIZE(code); + /* coverity[ampersand_in_size] */ + self = PyObject_NEW_VAR(PatternObject, &Pattern_Type, n); + if (!self) + return NULL; + self->weakreflist = NULL; + self->pattern = NULL; + self->groupindex = NULL; + self->indexgroup = NULL; + + self->codesize = n; + + for (i = 0; i < n; i++) { + PyObject *o = PyList_GET_ITEM(code, i); + unsigned long value = PyInt_Check(o) ? (unsigned long)PyInt_AsLong(o) + : PyLong_AsUnsignedLong(o); + self->code[i] = (SRE_CODE) value; + if ((unsigned long) self->code[i] != value) { + PyErr_SetString(PyExc_OverflowError, + "regular expression code size limit exceeded"); + break; + } + } + + if (PyErr_Occurred()) { + Py_DECREF(self); + return NULL; + } + + Py_INCREF(pattern); + self->pattern = pattern; + + self->flags = flags; + + self->groups = groups; + + Py_XINCREF(groupindex); + self->groupindex = groupindex; + + Py_XINCREF(indexgroup); + self->indexgroup = indexgroup; + + self->weakreflist = NULL; + + if (!_validate(self)) { + Py_DECREF(self); + return NULL; + } + + return (PyObject*) self; +} + +/* -------------------------------------------------------------------- */ +/* Code validation */ + +/* To learn more about this code, have a look at the _compile() function in + Lib/sre_compile.py. The validation functions below checks the code array + for conformance with the code patterns generated there. + + The nice thing about the generated code is that it is position-independent: + all jumps are relative jumps forward. Also, jumps don't cross each other: + the target of a later jump is always earlier than the target of an earlier + jump. IOW, this is okay: + + J---------J-------T--------T + \ \_____/ / + \______________________/ + + but this is not: + + J---------J-------T--------T + \_________\_____/ / + \____________/ + + It also helps that SRE_CODE is always an unsigned type, either 2 bytes or 4 + bytes wide (the latter if Python is compiled for "wide" unicode support). +*/ + +/* Defining this one enables tracing of the validator */ +#undef VVERBOSE + +/* Trace macro for the validator */ +#if defined(VVERBOSE) +#define VTRACE(v) printf v +#else +#define VTRACE(v) +#endif + +/* Report failure */ +#define FAIL do { VTRACE(("FAIL: %d\n", __LINE__)); return 0; } while (0) + +/* Extract opcode, argument, or skip count from code array */ +#define GET_OP \ + do { \ + VTRACE(("%p: ", code)); \ + if (code >= end) FAIL; \ + op = *code++; \ + VTRACE(("%lu (op)\n", (unsigned long)op)); \ + } while (0) +#define GET_ARG \ + do { \ + VTRACE(("%p= ", code)); \ + if (code >= end) FAIL; \ + arg = *code++; \ + VTRACE(("%lu (arg)\n", (unsigned long)arg)); \ + } while (0) +#define GET_SKIP_ADJ(adj) \ + do { \ + VTRACE(("%p= ", code)); \ + if (code >= end) FAIL; \ + skip = *code; \ + VTRACE(("%lu (skip to %p)\n", \ + (unsigned long)skip, code+skip)); \ + if (code+skip-adj < code || code+skip-adj > end)\ + FAIL; \ + code++; \ + } while (0) +#define GET_SKIP GET_SKIP_ADJ(0) + +static int +_validate_charset(SRE_CODE *code, SRE_CODE *end) +{ + /* Some variables are manipulated by the macros above */ + SRE_CODE op; + SRE_CODE arg; + SRE_CODE offset; + int i; + + while (code < end) { + GET_OP; + switch (op) { + + case SRE_OP_NEGATE: + break; + + case SRE_OP_LITERAL: + GET_ARG; + break; + + case SRE_OP_RANGE: + GET_ARG; + GET_ARG; + break; + + case SRE_OP_CHARSET: + offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */ + if (code+offset < code || code+offset > end) + FAIL; + code += offset; + break; + + case SRE_OP_BIGCHARSET: + GET_ARG; /* Number of blocks */ + offset = 256/sizeof(SRE_CODE); /* 256-byte table */ + if (code+offset < code || code+offset > end) + FAIL; + /* Make sure that each byte points to a valid block */ + for (i = 0; i < 256; i++) { + if (((unsigned char *)code)[i] >= arg) + FAIL; + } + code += offset; + offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */ + if (code+offset < code || code+offset > end) + FAIL; + code += offset; + break; + + case SRE_OP_CATEGORY: + GET_ARG; + switch (arg) { + case SRE_CATEGORY_DIGIT: + case SRE_CATEGORY_NOT_DIGIT: + case SRE_CATEGORY_SPACE: + case SRE_CATEGORY_NOT_SPACE: + case SRE_CATEGORY_WORD: + case SRE_CATEGORY_NOT_WORD: + case SRE_CATEGORY_LINEBREAK: + case SRE_CATEGORY_NOT_LINEBREAK: + case SRE_CATEGORY_LOC_WORD: + case SRE_CATEGORY_LOC_NOT_WORD: + case SRE_CATEGORY_UNI_DIGIT: + case SRE_CATEGORY_UNI_NOT_DIGIT: + case SRE_CATEGORY_UNI_SPACE: + case SRE_CATEGORY_UNI_NOT_SPACE: + case SRE_CATEGORY_UNI_WORD: + case SRE_CATEGORY_UNI_NOT_WORD: + case SRE_CATEGORY_UNI_LINEBREAK: + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + break; + default: + FAIL; + } + break; + + default: + FAIL; + + } + } + + return 1; +} + +static int +_validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +{ + /* Some variables are manipulated by the macros above */ + SRE_CODE op; + SRE_CODE arg; + SRE_CODE skip; + + VTRACE(("code=%p, end=%p\n", code, end)); + + if (code > end) + FAIL; + + while (code < end) { + GET_OP; + switch (op) { + + case SRE_OP_MARK: + /* We don't check whether marks are properly nested; the + sre_match() code is robust even if they don't, and the worst + you can get is nonsensical match results. */ + GET_ARG; + if (arg > 2*groups+1) { + VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); + FAIL; + } + break; + + case SRE_OP_LITERAL: + case SRE_OP_NOT_LITERAL: + case SRE_OP_LITERAL_IGNORE: + case SRE_OP_NOT_LITERAL_IGNORE: + GET_ARG; + /* The arg is just a character, nothing to check */ + break; + + case SRE_OP_SUCCESS: + case SRE_OP_FAILURE: + /* Nothing to check; these normally end the matching process */ + break; + + case SRE_OP_AT: + GET_ARG; + switch (arg) { + case SRE_AT_BEGINNING: + case SRE_AT_BEGINNING_STRING: + case SRE_AT_BEGINNING_LINE: + case SRE_AT_END: + case SRE_AT_END_LINE: + case SRE_AT_END_STRING: + case SRE_AT_BOUNDARY: + case SRE_AT_NON_BOUNDARY: + case SRE_AT_LOC_BOUNDARY: + case SRE_AT_LOC_NON_BOUNDARY: + case SRE_AT_UNI_BOUNDARY: + case SRE_AT_UNI_NON_BOUNDARY: + break; + default: + FAIL; + } + break; + + case SRE_OP_ANY: + case SRE_OP_ANY_ALL: + /* These have no operands */ + break; + + case SRE_OP_IN: + case SRE_OP_IN_IGNORE: + GET_SKIP; + /* Stop 1 before the end; we check the FAILURE below */ + if (!_validate_charset(code, code+skip-2)) + FAIL; + if (code[skip-2] != SRE_OP_FAILURE) + FAIL; + code += skip-1; + break; + + case SRE_OP_INFO: + { + /* A minimal info field is + <1=skip> <2=flags> <3=min> <4=max>; + If SRE_INFO_PREFIX or SRE_INFO_CHARSET is in the flags, + more follows. */ + SRE_CODE flags, min, max, i; + SRE_CODE *newcode; + GET_SKIP; + newcode = code+skip-1; + GET_ARG; flags = arg; + GET_ARG; min = arg; + GET_ARG; max = arg; + /* Check that only valid flags are present */ + if ((flags & ~(SRE_INFO_PREFIX | + SRE_INFO_LITERAL | + SRE_INFO_CHARSET)) != 0) + FAIL; + /* PREFIX and CHARSET are mutually exclusive */ + if ((flags & SRE_INFO_PREFIX) && + (flags & SRE_INFO_CHARSET)) + FAIL; + /* LITERAL implies PREFIX */ + if ((flags & SRE_INFO_LITERAL) && + !(flags & SRE_INFO_PREFIX)) + FAIL; + /* Validate the prefix */ + if (flags & SRE_INFO_PREFIX) { + SRE_CODE prefix_len, prefix_skip; + GET_ARG; prefix_len = arg; + GET_ARG; prefix_skip = arg; + /* Here comes the prefix string */ + if (code+prefix_len < code || code+prefix_len > newcode) + FAIL; + code += prefix_len; + /* And here comes the overlap table */ + if (code+prefix_len < code || code+prefix_len > newcode) + FAIL; + /* Each overlap value should be < prefix_len */ + for (i = 0; i < prefix_len; i++) { + if (code[i] >= prefix_len) + FAIL; + } + code += prefix_len; + } + /* Validate the charset */ + if (flags & SRE_INFO_CHARSET) { + if (!_validate_charset(code, newcode-1)) + FAIL; + if (newcode[-1] != SRE_OP_FAILURE) + FAIL; + code = newcode; + } + else if (code != newcode) { + VTRACE(("code=%p, newcode=%p\n", code, newcode)); + FAIL; + } + } + break; + + case SRE_OP_BRANCH: + { + SRE_CODE *target = NULL; + for (;;) { + GET_SKIP; + if (skip == 0) + break; + /* Stop 2 before the end; we check the JUMP below */ + if (!_validate_inner(code, code+skip-3, groups)) + FAIL; + code += skip-3; + /* Check that it ends with a JUMP, and that each JUMP + has the same target */ + GET_OP; + if (op != SRE_OP_JUMP) + FAIL; + GET_SKIP; + if (target == NULL) + target = code+skip-1; + else if (code+skip-1 != target) + FAIL; + } + } + break; + + case SRE_OP_REPEAT_ONE: + case SRE_OP_MIN_REPEAT_ONE: + { + SRE_CODE min, max; + GET_SKIP; + GET_ARG; min = arg; + GET_ARG; max = arg; + if (min > max) + FAIL; +#ifdef Py_UNICODE_WIDE + if (max > 65535) + FAIL; +#endif + if (!_validate_inner(code, code+skip-4, groups)) + FAIL; + code += skip-4; + GET_OP; + if (op != SRE_OP_SUCCESS) + FAIL; + } + break; + + case SRE_OP_REPEAT: + { + SRE_CODE min, max; + GET_SKIP; + GET_ARG; min = arg; + GET_ARG; max = arg; + if (min > max) + FAIL; +#ifdef Py_UNICODE_WIDE + if (max > 65535) + FAIL; +#endif + if (!_validate_inner(code, code+skip-3, groups)) + FAIL; + code += skip-3; + GET_OP; + if (op != SRE_OP_MAX_UNTIL && op != SRE_OP_MIN_UNTIL) + FAIL; + } + break; + + case SRE_OP_GROUPREF: + case SRE_OP_GROUPREF_IGNORE: + GET_ARG; + if (arg >= groups) + FAIL; + break; + + case SRE_OP_GROUPREF_EXISTS: + /* The regex syntax for this is: '(?(group)then|else)', where + 'group' is either an integer group number or a group name, + 'then' and 'else' are sub-regexes, and 'else' is optional. */ + GET_ARG; + if (arg >= groups) + FAIL; + GET_SKIP_ADJ(1); + code--; /* The skip is relative to the first arg! */ + /* There are two possibilities here: if there is both a 'then' + part and an 'else' part, the generated code looks like: + + GROUPREF_EXISTS + + + ...then part... + JUMP + + ( jumps here) + ...else part... + ( jumps here) + + If there is only a 'then' part, it looks like: + + GROUPREF_EXISTS + + + ...then part... + ( jumps here) + + There is no direct way to decide which it is, and we don't want + to allow arbitrary jumps anywhere in the code; so we just look + for a JUMP opcode preceding our skip target. + */ + if (skip >= 3 && code+skip-3 >= code && + code[skip-3] == SRE_OP_JUMP) + { + VTRACE(("both then and else parts present\n")); + if (!_validate_inner(code+1, code+skip-3, groups)) + FAIL; + code += skip-2; /* Position after JUMP, at */ + GET_SKIP; + if (!_validate_inner(code, code+skip-1, groups)) + FAIL; + code += skip-1; + } + else { + VTRACE(("only a then part present\n")); + if (!_validate_inner(code+1, code+skip-1, groups)) + FAIL; + code += skip-1; + } + break; + + case SRE_OP_ASSERT: + case SRE_OP_ASSERT_NOT: + GET_SKIP; + GET_ARG; /* 0 for lookahead, width for lookbehind */ + code--; /* Back up over arg to simplify math below */ + if (arg & 0x80000000) + FAIL; /* Width too large */ + /* Stop 1 before the end; we check the SUCCESS below */ + if (!_validate_inner(code+1, code+skip-2, groups)) + FAIL; + code += skip-2; + GET_OP; + if (op != SRE_OP_SUCCESS) + FAIL; + break; + + default: + FAIL; + + } + } + + VTRACE(("okay\n")); + return 1; +} + +static int +_validate_outer(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) +{ + if (groups < 0 || groups > 100 || code >= end || end[-1] != SRE_OP_SUCCESS) + FAIL; + if (groups == 0) /* fix for simplejson */ + groups = 100; /* 100 groups should always be safe */ + return _validate_inner(code, end-1, groups); +} + +static int +_validate(PatternObject *self) +{ + if (!_validate_outer(self->code, self->code+self->codesize, self->groups)) + { + PyErr_SetString(PyExc_RuntimeError, "invalid SRE code"); + return 0; + } + else + VTRACE(("Success!\n")); + return 1; +} + +/* -------------------------------------------------------------------- */ +/* match methods */ + +static void +match_dealloc(MatchObject* self) +{ + Py_XDECREF(self->regs); + Py_XDECREF(self->string); + Py_DECREF(self->pattern); + PyObject_DEL(self); +} + +static PyObject* +match_getslice_by_index(MatchObject* self, Py_ssize_t index, PyObject* def) +{ + if (index < 0 || index >= self->groups) { + /* raise IndexError if we were given a bad group number */ + PyErr_SetString( + PyExc_IndexError, + "no such group" + ); + return NULL; + } + + index *= 2; + + if (self->string == Py_None || self->mark[index] < 0) { + /* return default value if the string or group is undefined */ + Py_INCREF(def); + return def; + } + + return PySequence_GetSlice( + self->string, self->mark[index], self->mark[index+1] + ); +} + +static Py_ssize_t +match_getindex(MatchObject* self, PyObject* index) +{ + Py_ssize_t i; + + if (PyInt_Check(index)) + return PyInt_AsSsize_t(index); + + i = -1; + + if (self->pattern->groupindex) { + index = PyObject_GetItem(self->pattern->groupindex, index); + if (index) { + if (PyInt_Check(index) || PyLong_Check(index)) + i = PyInt_AsSsize_t(index); + Py_DECREF(index); + } else + PyErr_Clear(); + } + + return i; +} + +static PyObject* +match_getslice(MatchObject* self, PyObject* index, PyObject* def) +{ + return match_getslice_by_index(self, match_getindex(self, index), def); +} + +static PyObject* +match_expand(MatchObject* self, PyObject* ptemplate) +{ + /* delegate to Python code */ + return call( + SRE_PY_MODULE, "_expand", + PyTuple_Pack(3, self->pattern, self, ptemplate) + ); +} + +static PyObject* +match_group(MatchObject* self, PyObject* args) +{ + PyObject* result; + Py_ssize_t i, size; + + size = PyTuple_GET_SIZE(args); + + switch (size) { + case 0: + result = match_getslice(self, Py_False, Py_None); + break; + case 1: + result = match_getslice(self, PyTuple_GET_ITEM(args, 0), Py_None); + break; + default: + /* fetch multiple items */ + result = PyTuple_New(size); + if (!result) + return NULL; + for (i = 0; i < size; i++) { + PyObject* item = match_getslice( + self, PyTuple_GET_ITEM(args, i), Py_None + ); + if (!item) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(result, i, item); + } + break; + } + return result; +} + +static PyObject* +match_groups(MatchObject* self, PyObject* args, PyObject* kw) +{ + PyObject* result; + Py_ssize_t index; + + PyObject* def = Py_None; + static char* kwlist[] = { "default", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:groups", kwlist, &def)) + return NULL; + + result = PyTuple_New(self->groups-1); + if (!result) + return NULL; + + for (index = 1; index < self->groups; index++) { + PyObject* item; + item = match_getslice_by_index(self, index, def); + if (!item) { + Py_DECREF(result); + return NULL; + } + PyTuple_SET_ITEM(result, index-1, item); + } + + return result; +} + +static PyObject* +match_groupdict(MatchObject* self, PyObject* args, PyObject* kw) +{ + PyObject* result; + PyObject* keys; + Py_ssize_t index; + + PyObject* def = Py_None; + static char* kwlist[] = { "default", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:groupdict", kwlist, &def)) + return NULL; + + result = PyDict_New(); + if (!result || !self->pattern->groupindex) + return result; + + keys = PyMapping_Keys(self->pattern->groupindex); + if (!keys) + goto failed; + + for (index = 0; index < PyList_GET_SIZE(keys); index++) { + int status; + PyObject* key; + PyObject* value; + key = PyList_GET_ITEM(keys, index); + if (!key) + goto failed; + value = match_getslice(self, key, def); + if (!value) { + Py_DECREF(key); + goto failed; + } + status = PyDict_SetItem(result, key, value); + Py_DECREF(value); + if (status < 0) + goto failed; + } + + Py_DECREF(keys); + + return result; + +failed: + Py_XDECREF(keys); + Py_DECREF(result); + return NULL; +} + +static PyObject* +match_start(MatchObject* self, PyObject* args) +{ + Py_ssize_t index; + + PyObject* index_ = Py_False; /* zero */ + if (!PyArg_UnpackTuple(args, "start", 0, 1, &index_)) + return NULL; + + index = match_getindex(self, index_); + + if (index < 0 || index >= self->groups) { + PyErr_SetString( + PyExc_IndexError, + "no such group" + ); + return NULL; + } + + /* mark is -1 if group is undefined */ + return Py_BuildValue("i", self->mark[index*2]); +} + +static PyObject* +match_end(MatchObject* self, PyObject* args) +{ + Py_ssize_t index; + + PyObject* index_ = Py_False; /* zero */ + if (!PyArg_UnpackTuple(args, "end", 0, 1, &index_)) + return NULL; + + index = match_getindex(self, index_); + + if (index < 0 || index >= self->groups) { + PyErr_SetString( + PyExc_IndexError, + "no such group" + ); + return NULL; + } + + /* mark is -1 if group is undefined */ + return Py_BuildValue("i", self->mark[index*2+1]); +} + +LOCAL(PyObject*) +_pair(Py_ssize_t i1, Py_ssize_t i2) +{ + PyObject* pair; + PyObject* item; + + pair = PyTuple_New(2); + if (!pair) + return NULL; + + item = PyInt_FromSsize_t(i1); + if (!item) + goto error; + PyTuple_SET_ITEM(pair, 0, item); + + item = PyInt_FromSsize_t(i2); + if (!item) + goto error; + PyTuple_SET_ITEM(pair, 1, item); + + return pair; + + error: + Py_DECREF(pair); + return NULL; +} + +static PyObject* +match_span(MatchObject* self, PyObject* args) +{ + Py_ssize_t index; + + PyObject* index_ = Py_False; /* zero */ + if (!PyArg_UnpackTuple(args, "span", 0, 1, &index_)) + return NULL; + + index = match_getindex(self, index_); + + if (index < 0 || index >= self->groups) { + PyErr_SetString( + PyExc_IndexError, + "no such group" + ); + return NULL; + } + + /* marks are -1 if group is undefined */ + return _pair(self->mark[index*2], self->mark[index*2+1]); +} + +static PyObject* +match_regs(MatchObject* self) +{ + PyObject* regs; + PyObject* item; + Py_ssize_t index; + + regs = PyTuple_New(self->groups); + if (!regs) + return NULL; + + for (index = 0; index < self->groups; index++) { + item = _pair(self->mark[index*2], self->mark[index*2+1]); + if (!item) { + Py_DECREF(regs); + return NULL; + } + PyTuple_SET_ITEM(regs, index, item); + } + + Py_INCREF(regs); + self->regs = regs; + + return regs; +} + +static PyObject* +match_copy(MatchObject* self, PyObject *unused) +{ +#ifdef USE_BUILTIN_COPY + MatchObject* copy; + Py_ssize_t slots, offset; + + slots = 2 * (self->pattern->groups+1); + + copy = PyObject_NEW_VAR(MatchObject, &Match_Type, slots); + if (!copy) + return NULL; + + /* this value a constant, but any compiler should be able to + figure that out all by itself */ + offset = offsetof(MatchObject, string); + + Py_XINCREF(self->pattern); + Py_XINCREF(self->string); + Py_XINCREF(self->regs); + + memcpy((char*) copy + offset, (char*) self + offset, + sizeof(MatchObject) + slots * sizeof(Py_ssize_t) - offset); + + return (PyObject*) copy; +#else + PyErr_SetString(PyExc_TypeError, "cannot copy this match object"); + return NULL; +#endif +} + +static PyObject* +match_deepcopy(MatchObject* self, PyObject* memo) +{ +#ifdef USE_BUILTIN_COPY + MatchObject* copy; + + copy = (MatchObject*) match_copy(self); + if (!copy) + return NULL; + + if (!deepcopy((PyObject**) ©->pattern, memo) || + !deepcopy(©->string, memo) || + !deepcopy(©->regs, memo)) { + Py_DECREF(copy); + return NULL; + } + +#else + PyErr_SetString(PyExc_TypeError, "cannot deepcopy this match object"); + return NULL; +#endif +} + +static PyMethodDef match_methods[] = { + {"group", (PyCFunction) match_group, METH_VARARGS}, + {"start", (PyCFunction) match_start, METH_VARARGS}, + {"end", (PyCFunction) match_end, METH_VARARGS}, + {"span", (PyCFunction) match_span, METH_VARARGS}, + {"groups", (PyCFunction) match_groups, METH_VARARGS|METH_KEYWORDS}, + {"groupdict", (PyCFunction) match_groupdict, METH_VARARGS|METH_KEYWORDS}, + {"expand", (PyCFunction) match_expand, METH_O}, + {"__copy__", (PyCFunction) match_copy, METH_NOARGS}, + {"__deepcopy__", (PyCFunction) match_deepcopy, METH_O}, + {NULL, NULL} +}; + +static PyObject* +match_getattr(MatchObject* self, char* name) +{ + PyObject* res; + + res = Py_FindMethod(match_methods, (PyObject*) self, name); + if (res) + return res; + + PyErr_Clear(); + + if (!strcmp(name, "lastindex")) { + if (self->lastindex >= 0) + return Py_BuildValue("i", self->lastindex); + Py_INCREF(Py_None); + return Py_None; + } + + if (!strcmp(name, "lastgroup")) { + if (self->pattern->indexgroup && self->lastindex >= 0) { + PyObject* result = PySequence_GetItem( + self->pattern->indexgroup, self->lastindex + ); + if (result) + return result; + PyErr_Clear(); + } + Py_INCREF(Py_None); + return Py_None; + } + + if (!strcmp(name, "string")) { + if (self->string) { + Py_INCREF(self->string); + return self->string; + } else { + Py_INCREF(Py_None); + return Py_None; + } + } + + if (!strcmp(name, "regs")) { + if (self->regs) { + Py_INCREF(self->regs); + return self->regs; + } else + return match_regs(self); + } + + if (!strcmp(name, "re")) { + Py_INCREF(self->pattern); + return (PyObject*) self->pattern; + } + + if (!strcmp(name, "pos")) + return Py_BuildValue("i", self->pos); + + if (!strcmp(name, "endpos")) + return Py_BuildValue("i", self->endpos); + + PyErr_SetString(PyExc_AttributeError, name); + return NULL; +} + +/* FIXME: implement setattr("string", None) as a special case (to + detach the associated string, if any */ + +statichere PyTypeObject Match_Type = { + PyObject_HEAD_INIT(NULL) + 0, "_" SRE_MODULE ".SRE_Match", + sizeof(MatchObject), sizeof(Py_ssize_t), + (destructor)match_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)match_getattr /*tp_getattr*/ +}; + +static PyObject* +pattern_new_match(PatternObject* pattern, SRE_STATE* state, int status) +{ + /* create match object (from state object) */ + + MatchObject* match; + Py_ssize_t i, j; + char* base; + int n; + + if (status > 0) { + + /* create match object (with room for extra group marks) */ + /* coverity[ampersand_in_size] */ + match = PyObject_NEW_VAR(MatchObject, &Match_Type, + 2*(pattern->groups+1)); + if (!match) + return NULL; + + Py_INCREF(pattern); + match->pattern = pattern; + + Py_INCREF(state->string); + match->string = state->string; + + match->regs = NULL; + match->groups = pattern->groups+1; + + /* fill in group slices */ + + base = (char*) state->beginning; + n = state->charsize; + + match->mark[0] = ((char*) state->start - base) / n; + match->mark[1] = ((char*) state->ptr - base) / n; + + for (i = j = 0; i < pattern->groups; i++, j+=2) + if (j+1 <= state->lastmark && state->mark[j] && state->mark[j+1]) { + match->mark[j+2] = ((char*) state->mark[j] - base) / n; + match->mark[j+3] = ((char*) state->mark[j+1] - base) / n; + } else + match->mark[j+2] = match->mark[j+3] = -1; /* undefined */ + + match->pos = state->pos; + match->endpos = state->endpos; + + match->lastindex = state->lastindex; + + return (PyObject*) match; + + } else if (status == 0) { + + /* no match */ + Py_INCREF(Py_None); + return Py_None; + + } + + /* internal error */ + pattern_error(status); + return NULL; +} + + +/* -------------------------------------------------------------------- */ +/* scanner methods (experimental) */ + +static void +scanner_dealloc(ScannerObject* self) +{ + state_fini(&self->state); + Py_XDECREF(self->pattern); + PyObject_DEL(self); +} + +static PyObject* +scanner_match(ScannerObject* self, PyObject *unused) +{ + SRE_STATE* state = &self->state; + PyObject* match; + int status; + + state_reset(state); + + state->ptr = state->start; + + if (state->charsize == 1) { + status = sre_match(state, PatternObject_GetCode(self->pattern)); + } else { +#if defined(HAVE_UNICODE) + status = sre_umatch(state, PatternObject_GetCode(self->pattern)); +#endif + } + if (PyErr_Occurred()) + return NULL; + + match = pattern_new_match((PatternObject*) self->pattern, + state, status); + + if (status == 0 || state->ptr == state->start) + state->start = (void*) ((char*) state->ptr + state->charsize); + else + state->start = state->ptr; + + return match; +} + + +static PyObject* +scanner_search(ScannerObject* self, PyObject *unused) +{ + SRE_STATE* state = &self->state; + PyObject* match; + int status; + + state_reset(state); + + state->ptr = state->start; + + if (state->charsize == 1) { + status = sre_search(state, PatternObject_GetCode(self->pattern)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(state, PatternObject_GetCode(self->pattern)); +#endif + } + if (PyErr_Occurred()) + return NULL; + + match = pattern_new_match((PatternObject*) self->pattern, + state, status); + + if (status == 0 || state->ptr == state->start) + state->start = (void*) ((char*) state->ptr + state->charsize); + else + state->start = state->ptr; + + return match; +} + +static PyMethodDef scanner_methods[] = { + {"match", (PyCFunction) scanner_match, METH_NOARGS}, + {"search", (PyCFunction) scanner_search, METH_NOARGS}, + {NULL, NULL} +}; + +static PyObject* +scanner_getattr(ScannerObject* self, char* name) +{ + PyObject* res; + + res = Py_FindMethod(scanner_methods, (PyObject*) self, name); + if (res) + return res; + + PyErr_Clear(); + + /* attributes */ + if (!strcmp(name, "pattern")) { + Py_INCREF(self->pattern); + return self->pattern; + } + + PyErr_SetString(PyExc_AttributeError, name); + return NULL; +} + +statichere PyTypeObject Scanner_Type = { + PyObject_HEAD_INIT(NULL) + 0, "_" SRE_MODULE ".SRE_Scanner", + sizeof(ScannerObject), 0, + (destructor)scanner_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)scanner_getattr, /*tp_getattr*/ +}; + +static PyObject* +pattern_scanner(PatternObject* pattern, PyObject* args) +{ + /* create search state object */ + + ScannerObject* self; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + if (!PyArg_ParseTuple(args, "O|nn:scanner", &string, &start, &end)) + return NULL; + + /* create scanner object */ + self = PyObject_NEW(ScannerObject, &Scanner_Type); + if (!self) + return NULL; + self->pattern = NULL; + + string = state_init(&self->state, pattern, string, start, end); + if (!string) { + Py_DECREF(self); + return NULL; + } + + Py_INCREF(pattern); + self->pattern = (PyObject*) pattern; + + return (PyObject*) self; +} + +static PyMethodDef _functions[] = { + {"compile", _compile, METH_VARARGS}, + {"getcodesize", sre_codesize, METH_NOARGS}, + {"getlower", sre_getlower, METH_VARARGS}, + {NULL, NULL} +}; + +#if PY_VERSION_HEX < 0x02030000 +DL_EXPORT(void) init_sre(void) +#else +PyMODINIT_FUNC init_sre(void) +#endif +{ + PyObject* m; + PyObject* d; + PyObject* x; + + /* Patch object types */ + Pattern_Type.ob_type = Match_Type.ob_type = + Scanner_Type.ob_type = &PyType_Type; + + m = Py_InitModule("_" SRE_MODULE, _functions); + if (m == NULL) + return; + d = PyModule_GetDict(m); + + x = PyInt_FromLong(SRE_MAGIC); + if (x) { + PyDict_SetItemString(d, "MAGIC", x); + Py_DECREF(x); + } + + x = PyInt_FromLong(sizeof(SRE_CODE)); + if (x) { + PyDict_SetItemString(d, "CODESIZE", x); + Py_DECREF(x); + } + + x = PyString_FromString(copyright); + if (x) { + PyDict_SetItemString(d, "copyright", x); + Py_DECREF(x); + } +} + +#endif /* !defined(SRE_RECURSIVE) */ + +/* vim:ts=4:sw=4:et +*/ Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/sre.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/sre.h Sun Apr 4 19:14:58 2010 @@ -0,0 +1,94 @@ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the _sre.c file for information on usage and redistribution. + */ + +#ifndef SRE_INCLUDED +#define SRE_INCLUDED + +#include "sre_constants.h" + +/* size of a code word (must be unsigned short or larger, and + large enough to hold a Py_UNICODE character) */ +#ifdef Py_UNICODE_WIDE +#define SRE_CODE Py_UCS4 +#else +#define SRE_CODE unsigned short +#endif + +typedef struct { + PyObject_VAR_HEAD + Py_ssize_t groups; /* must be first! */ + PyObject* groupindex; + PyObject* indexgroup; + /* compatibility */ + PyObject* pattern; /* pattern source (or None) */ + int flags; /* flags used when compiling pattern source */ + PyObject *weakreflist; /* List of weak references */ + /* pattern code */ + Py_ssize_t codesize; + SRE_CODE code[1]; +} PatternObject; + +#define PatternObject_GetCode(o) (((PatternObject*)(o))->code) + +typedef struct { + PyObject_VAR_HEAD + PyObject* string; /* link to the target string (must be first) */ + PyObject* regs; /* cached list of matching spans */ + PatternObject* pattern; /* link to the regex (pattern) object */ + Py_ssize_t pos, endpos; /* current target slice */ + Py_ssize_t lastindex; /* last index marker seen by the engine (-1 if none) */ + Py_ssize_t groups; /* number of groups (start/end marks) */ + Py_ssize_t mark[1]; +} MatchObject; + +typedef unsigned int (*SRE_TOLOWER_HOOK)(unsigned int ch); + +/* FIXME: shouldn't be a constant, really... */ +#define SRE_MARK_SIZE 200 + +typedef struct SRE_REPEAT_T { + Py_ssize_t count; + SRE_CODE* pattern; /* points to REPEAT operator arguments */ + void* last_ptr; /* helper to check for infinite loops */ + struct SRE_REPEAT_T *prev; /* points to previous repeat context */ +} SRE_REPEAT; + +typedef struct { + /* string pointers */ + void* ptr; /* current position (also end of current slice) */ + void* beginning; /* start of original string */ + void* start; /* start of current slice */ + void* end; /* end of original string */ + /* attributes for the match object */ + PyObject* string; + Py_ssize_t pos, endpos; + /* character size */ + int charsize; + /* registers */ + Py_ssize_t lastindex; + Py_ssize_t lastmark; + void* mark[SRE_MARK_SIZE]; + /* dynamically allocated stuff */ + char* data_stack; + size_t data_stack_size; + size_t data_stack_base; + /* current repeat context */ + SRE_REPEAT *repeat; + /* hooks */ + SRE_TOLOWER_HOOK lower; +} SRE_STATE; + +typedef struct { + PyObject_HEAD + PyObject* pattern; + SRE_STATE state; +} ScannerObject; + +#endif Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/sre_constants.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/sre_constants.h Sun Apr 4 19:14:58 2010 @@ -0,0 +1,86 @@ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * NOTE: This file is generated by sre_constants.py. If you need + * to change anything in here, edit sre_constants.py and run it. + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the _sre.c file for information on usage and redistribution. + */ + +#define SRE_MAGIC 20031017 +#define SRE_OP_FAILURE 0 +#define SRE_OP_SUCCESS 1 +#define SRE_OP_ANY 2 +#define SRE_OP_ANY_ALL 3 +#define SRE_OP_ASSERT 4 +#define SRE_OP_ASSERT_NOT 5 +#define SRE_OP_AT 6 +#define SRE_OP_BRANCH 7 +#define SRE_OP_CALL 8 +#define SRE_OP_CATEGORY 9 +#define SRE_OP_CHARSET 10 +#define SRE_OP_BIGCHARSET 11 +#define SRE_OP_GROUPREF 12 +#define SRE_OP_GROUPREF_EXISTS 13 +#define SRE_OP_GROUPREF_IGNORE 14 +#define SRE_OP_IN 15 +#define SRE_OP_IN_IGNORE 16 +#define SRE_OP_INFO 17 +#define SRE_OP_JUMP 18 +#define SRE_OP_LITERAL 19 +#define SRE_OP_LITERAL_IGNORE 20 +#define SRE_OP_MARK 21 +#define SRE_OP_MAX_UNTIL 22 +#define SRE_OP_MIN_UNTIL 23 +#define SRE_OP_NOT_LITERAL 24 +#define SRE_OP_NOT_LITERAL_IGNORE 25 +#define SRE_OP_NEGATE 26 +#define SRE_OP_RANGE 27 +#define SRE_OP_REPEAT 28 +#define SRE_OP_REPEAT_ONE 29 +#define SRE_OP_SUBPATTERN 30 +#define SRE_OP_MIN_REPEAT_ONE 31 +#define SRE_AT_BEGINNING 0 +#define SRE_AT_BEGINNING_LINE 1 +#define SRE_AT_BEGINNING_STRING 2 +#define SRE_AT_BOUNDARY 3 +#define SRE_AT_NON_BOUNDARY 4 +#define SRE_AT_END 5 +#define SRE_AT_END_LINE 6 +#define SRE_AT_END_STRING 7 +#define SRE_AT_LOC_BOUNDARY 8 +#define SRE_AT_LOC_NON_BOUNDARY 9 +#define SRE_AT_UNI_BOUNDARY 10 +#define SRE_AT_UNI_NON_BOUNDARY 11 +#define SRE_CATEGORY_DIGIT 0 +#define SRE_CATEGORY_NOT_DIGIT 1 +#define SRE_CATEGORY_SPACE 2 +#define SRE_CATEGORY_NOT_SPACE 3 +#define SRE_CATEGORY_WORD 4 +#define SRE_CATEGORY_NOT_WORD 5 +#define SRE_CATEGORY_LINEBREAK 6 +#define SRE_CATEGORY_NOT_LINEBREAK 7 +#define SRE_CATEGORY_LOC_WORD 8 +#define SRE_CATEGORY_LOC_NOT_WORD 9 +#define SRE_CATEGORY_UNI_DIGIT 10 +#define SRE_CATEGORY_UNI_NOT_DIGIT 11 +#define SRE_CATEGORY_UNI_SPACE 12 +#define SRE_CATEGORY_UNI_NOT_SPACE 13 +#define SRE_CATEGORY_UNI_WORD 14 +#define SRE_CATEGORY_UNI_NOT_WORD 15 +#define SRE_CATEGORY_UNI_LINEBREAK 16 +#define SRE_CATEGORY_UNI_NOT_LINEBREAK 17 +#define SRE_FLAG_TEMPLATE 1 +#define SRE_FLAG_IGNORECASE 2 +#define SRE_FLAG_LOCALE 4 +#define SRE_FLAG_MULTILINE 8 +#define SRE_FLAG_DOTALL 16 +#define SRE_FLAG_UNICODE 32 +#define SRE_FLAG_VERBOSE 64 +#define SRE_INFO_PREFIX 1 +#define SRE_INFO_LITERAL 2 +#define SRE_INFO_CHARSET 4 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 19:14:58 2010 @@ -70,8 +70,6 @@ raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") - #skip("In progress") # not at all, how does this fail for you amaury? - a = module.fooType assert "cannot create" in raises(TypeError, "a()").value.message class bar(module.fooType): @@ -84,3 +82,7 @@ return self assert fuu2(u"abc").baz().escape() + def test_sre(self): + skip("In Progress") + module = self.import_module(name='_sre') + From xoraxax at codespeak.net Sun Apr 4 19:16:00 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:16:00 +0200 (CEST) Subject: [pypy-svn] r73361 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404171600.A3E67282BF9@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:15:59 2010 New Revision: 73361 Added: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Add PyImport_Import with a skipped failing test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Sun Apr 4 19:15:59 2010 @@ -47,6 +47,7 @@ import pypy.module.cpyext.sequence import pypy.module.cpyext.eval import pypy.module.cpyext.getargs +import pypy.module.cpyext.import_ # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/import_.py Sun Apr 4 19:15:59 2010 @@ -0,0 +1,29 @@ +from pypy.interpreter import module +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject +from pypy.interpreter.error import OperationError + + at cpython_api([PyObject], PyObject) +def PyImport_Import(space, w_name): + """ + This is a higher-level interface that calls the current "import hook function". + It invokes the __import__() function from the __builtins__ of the + current globals. This means that the import is done using whatever import hooks + are installed in the current environment, e.g. by rexec or ihooks. + + Always uses absolute imports.""" + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is not None: + w_globals = caller.w_globals + try: + w_builtin = space.getitem(w_globals, space.wrap('__builtins__')) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + else: + if space.is_true(space.isinstance(w_builtin, space.w_dict)): + w_builtin = module.Module(space, None, w_builtin) + builtin = space.interpclass_w(w_builtin) + if isinstance(builtin, module.Module): + return space.call(builtin.get("__import__"), space.newtuple([w_name])) + raise OperationError(space.w_KeyError, space.wrap("__builtins__")) + Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Sun Apr 4 19:15:59 2010 @@ -0,0 +1,11 @@ +import py + +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestImport(BaseApiTest): + def test_import(self, space, api): + # failing because we dont have a caller + skip("Fails currently, dont know how to fix") + pdb = api.PyImport_Import(space.wrap("pdb")) + assert pdb + assert pdb.get("pm") From xoraxax at codespeak.net Sun Apr 4 19:16:32 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:16:32 +0200 (CEST) Subject: [pypy-svn] r73362 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404171632.237F7282BFA@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:16:30 2010 New Revision: 73362 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: Add PyLong_AsUnsignedLong. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Sun Apr 4 19:16:30 2010 @@ -1,6 +1,7 @@ - -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.module.cpyext.api import cpython_api, PyObject +from pypy.objspace.std.longobject import W_LongObject +from pypy.interpreter.error import OperationError @cpython_api([lltype.Signed], PyObject) @@ -8,4 +9,11 @@ """Return a new PyLongObject object from v, or NULL on failure.""" return space.wrap(val) + at cpython_api([PyObject], rffi.ULONG, error=0) +def PyLong_AsUnsignedLong(space, w_long): + """ + Return a C unsigned long representation of the contents of pylong. + If pylong is greater than ULONG_MAX, an OverflowError is + raised.""" + return rffi.cast(rffi.ULONG, space.uint_w(w_long)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Sun Apr 4 19:16:30 2010 @@ -14,4 +14,9 @@ value = api.PyLong_FromLong(sys.maxint + 1) assert isinstance(value, W_LongObject) - assert space.unwrap(value) == sys.maxint + 1 + assert space.unwrap(value) == sys.maxint + 1 # should obviously fail but doesnt + def test_asulong(self, space, api): + w_value = api.PyLong_FromLong((sys.maxint - 1) / 2) + w_value = space.mul(w_value, space.wrap(4)) + value = api.PyLong_AsUnsignedLong(w_value) + assert value == (sys.maxint - 1) * 2 From xoraxax at codespeak.net Sun Apr 4 19:17:26 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:17:26 +0200 (CEST) Subject: [pypy-svn] r73363 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100404171726.02BF7282BFC@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:17:25 2010 New Revision: 73363 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Log: Add PyList_GET_ITEM define and PyList_GetItem impl. Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h Sun Apr 4 19:17:25 2010 @@ -0,0 +1 @@ +#define PyList_GET_ITEM PyList_GetItem Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Sun Apr 4 19:17:25 2010 @@ -39,6 +39,22 @@ wrappeditems[index] = w_item return 0 + at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) +def PyList_GetItem(space, w_list, index): + """Return the object at position pos in the list pointed to by p. The + position must be positive, indexing from the end of the list is not + supported. If pos is out of bounds, return NULL and set an + IndexError exception.""" + register_container(space, w_list) + if not isinstance(w_list, W_ListObject): + PyErr_BadInternalCall(space) + wrappeditems = w_list.wrappeditems + if index < 0 or index >= len(wrappeditems): + raise OperationError(space.w_IndexError, space.wrap( + "list index out of range")) + return wrappeditems[index] + + @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyList_Append(space, w_list, w_item): if not isinstance(w_list, W_ListObject): From xoraxax at codespeak.net Sun Apr 4 19:18:36 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:18:36 +0200 (CEST) Subject: [pypy-svn] r73364 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404171836.ED222282BFD@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:18:35 2010 New Revision: 73364 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Add PY_SSIZE_T_MAX and PY_SSIZE_T_MIN. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 4 19:18:35 2010 @@ -388,13 +388,28 @@ #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL -/* objimpl.h ----------------------------------------------*/ +/* objimpl.h ----------------------------------------------*/ +#define PyObject_DEL PyObject_Del #define PyObject_New(type, typeobj) \ ( (type *) _PyObject_New(typeobj) ) #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) +#define PyObject_INIT(op, typeobj) \ + ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) +#define PyObject_INIT_VAR(op, typeobj, size) \ + ( Py_SIZE(op) = (size), PyObject_INIT((op), (typeobj)) ) + +#define PyObject_NEW(type, typeobj) \ +( (type *) PyObject_Init( \ + (PyObject *) PyObject_MALLOC( _PyObject_SIZE(typeobj) ), (typeobj)) ) + +#define PyObject_NEW_VAR(type, typeobj, n) \ +( (type *) PyObject_InitVar( \ + (PyVarObject *) PyObject_MALLOC(_PyObject_VAR_SIZE((typeobj),(n)) ),\ + (typeobj), (n)) ) + /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); From xoraxax at codespeak.net Sun Apr 4 19:19:13 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:19:13 +0200 (CEST) Subject: [pypy-svn] r73365 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404171913.79C37282BFE@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:19:12 2010 New Revision: 73365 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Add various macros. Swap the commit message with the last rev. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 4 19:19:12 2010 @@ -27,6 +27,8 @@ # endif #endif #define Py_ssize_t long +#define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) +#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ @@ -63,6 +65,7 @@ #include "tupleobject.h" #include "dictobject.h" #include "intobject.h" +#include "listobject.h" #include "unicodeobject.h" #include "eval.h" #include "pymem.h" From xoraxax at codespeak.net Sun Apr 4 19:20:11 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:20:11 +0200 (CEST) Subject: [pypy-svn] r73366 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404172011.017E1282BFF@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:20:10 2010 New Revision: 73366 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Add various object methods. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 4 19:20:10 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL +from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ + Py_ssize_t from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State @@ -42,3 +43,18 @@ def PyObject_SetAttr(space, w_obj, w_name, w_value): operation.setattr(space, w_obj, w_name, w_value) return 0 + + at cpython_api([PyObject], lltype.Void) +def PyObject_ClearWeakRefs(space, w_object): + w_object.clear_all_weakrefs() + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyObject_Size(space, w_obj): + return space.len(w_obj) + + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyCallable_Check(space, w_obj): + """Determine if the object o is callable. Return 1 if the object is callable + and 0 otherwise. This function always succeeds.""" + return int(space.is_true(space.callable(w_obj))) + From xoraxax at codespeak.net Sun Apr 4 19:20:33 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:20:33 +0200 (CEST) Subject: [pypy-svn] r73367 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404172033.0C563282C00@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:20:31 2010 New Revision: 73367 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h Log: Add define for PyTuple_SET_ITEM. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h Sun Apr 4 19:20:31 2010 @@ -10,6 +10,8 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); +#define PyTuple_SET_ITEM PyTuple_SetItem + #ifdef __cplusplus } #endif From xoraxax at codespeak.net Sun Apr 4 19:21:12 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:21:12 +0200 (CEST) Subject: [pypy-svn] r73368 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404172112.7F75B282C01@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:21:11 2010 New Revision: 73368 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Log: Move stuff around and implement FindModule. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Sun Apr 4 19:21:11 2010 @@ -8,12 +8,24 @@ from pypy.interpreter.function import BuiltinFunction, Method from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref -from pypy.module.cpyext.api import generic_cpy_call +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ + cpython_struct from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated +PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) + +PyMethodDef = cpython_struct( + 'PyMethodDef', + [('ml_name', rffi.CCHARP), + ('ml_meth', PyCFunction), + ('ml_flags', rffi.INT_real), + ('ml_doc', rffi.CCHARP), + ]) + + class W_PyCFunctionObject(Wrappable): def __init__(self, space, ml, w_self, doc=None): self.ml = ml @@ -157,3 +169,24 @@ # not exactly the API sig return space.wrap(W_PyCWrapperObject(space, pto, method_name, wrapper_func, doc, flags, func)) + + at cpython_api([PyMethodDef, PyObject, rffi.CCHARP], PyObject) +def Py_FindMethod(space, table, w_ob, name_ptr): + """Return a bound method object for an extension type implemented in C. This + can be useful in the implementation of a tp_getattro or + tp_getattr handler that does not use the + PyObject_GenericGetAttr() function.""" + # XXX handle __doc__ + + name = rffi.charp2str(name_ptr) + methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) + if methods: + i = -1 + while True: + i = i + 1 + method = methods[i] + if not method.c_ml_name: break + if rffi.charp2str(method.c_ml_name) == name: # XXX expensive copying + return PyCFunction_NewEx(space, method, w_ob) + raise OperationError(space.w_AttributeError, space.wrap(name)) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Sun Apr 4 19:21:11 2010 @@ -3,20 +3,10 @@ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module -from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod +from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError -PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) - -PyMethodDef = cpython_struct( - 'PyMethodDef', - [('ml_name', rffi.CCHARP), - ('ml_meth', PyCFunction), - ('ml_flags', rffi.INT_real), - ('ml_doc', rffi.CCHARP), - ]) - def PyImport_AddModule(space, name): w_name = space.wrap(name) w_mod = space.wrap(Module(space, w_name)) @@ -101,10 +91,8 @@ @cpython_api([PyObject], rffi.CCHARP, error=0) def PyModule_GetName(space, module): """ - - - Return module's __name__ value. If the module does not provide one, or if it is not a string, SystemError is raised and NULL is returned.""" raise NotImplementedError + From xoraxax at codespeak.net Sun Apr 4 19:21:51 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:21:51 +0200 (CEST) Subject: [pypy-svn] r73369 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100404172151.AEDDD282C02@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:21:50 2010 New Revision: 73369 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/ (props changed) Log: Ignore *.errors. From agaynor at codespeak.net Sun Apr 4 19:36:19 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 19:36:19 +0200 (CEST) Subject: [pypy-svn] r73370 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404173619.93B23282C03@codespeak.net> Author: agaynor Date: Sun Apr 4 19:36:17 2010 New Revision: 73370 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Implement PYObject_GetItem. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 4 19:36:17 2010 @@ -58,3 +58,8 @@ and 0 otherwise. This function always succeeds.""" return int(space.is_true(space.callable(w_obj))) + at cpython_api([PyObject, PyObject], PyObject) +def PyObject_GetItem(space, w_obj, w_key): + """Return element of o corresponding to the object key or NULL on failure. + This is the equivalent of the Python expression o[key].""" + return space.getitem(w_obj, w_key) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 19:36:17 2010 @@ -4616,12 +4616,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyObject_GetItem(space, o, key): - """Return element of o corresponding to the object key or NULL on failure. - This is the equivalent of the Python expression o[key].""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real) def PyObject_SetItem(space, o, key, v): """Map the object key to the value v. Returns -1 on failure. This is the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sun Apr 4 19:36:17 2010 @@ -40,3 +40,11 @@ assert api.PyObject_HasAttr(space.wrap(x), space.wrap('test')) api.PyObject_SetAttr(space.wrap(x), space.wrap('test'), space.wrap(10)) assert x.test == 10 + + def test_getitem(self, space, api): + w_t = space.wrap((1, 2, 3, 4, 5)) + assert space.unwrap(api.PyObject_GetItem(w_t, space.wrap(3))) == 4 + + w_d = space.newdict() + space.setitem(w_d, space.wrap("a key!"), space.wrap(72)) + assert space.unwrap(api.PyObject_GetItem(w_d, space.wrap("a key!"))) == 72 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 19:36:17 2010 @@ -83,6 +83,5 @@ assert fuu2(u"abc").baz().escape() def test_sre(self): - skip("In Progress") module = self.import_module(name='_sre') From xoraxax at codespeak.net Sun Apr 4 19:36:58 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:36:58 +0200 (CEST) Subject: [pypy-svn] r73371 - pypy/branch/cpython-extension/pypy/translator/platform Message-ID: <20100404173658.F2A47282C04@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:36:57 2010 New Revision: 73371 Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Log: I want to see all errors, not only some! Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Sun Apr 4 19:36:57 2010 @@ -107,8 +107,8 @@ stderrlines = stderr.splitlines() for line in stderrlines[:50]: log.ERROR(line) - if len(stderrlines) > 50: - log.ERROR('...') + #if len(stderrlines) > 50: + # log.ERROR('...') raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): From agaynor at codespeak.net Sun Apr 4 19:37:05 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 19:37:05 +0200 (CEST) Subject: [pypy-svn] r73372 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100404173705.A00D6282C05@codespeak.net> Author: agaynor Date: Sun Apr 4 19:37:04 2010 New Revision: 73372 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: This shouldn't have gone in with the last commit. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 19:37:04 2010 @@ -83,5 +83,6 @@ assert fuu2(u"abc").baz().escape() def test_sre(self): + skip("In Progress") module = self.import_module(name='_sre') From xoraxax at codespeak.net Sun Apr 4 19:38:15 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:38:15 +0200 (CEST) Subject: [pypy-svn] r73373 - pypy/branch/cpython-extension/pypy/translator/platform Message-ID: <20100404173815.CFA6C282C06@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:38:14 2010 New Revision: 73373 Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Log: I want to see all errors, not only some! Pt. 2 Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Sun Apr 4 19:38:14 2010 @@ -105,10 +105,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() - for line in stderrlines[:50]: + for line in stderrlines: log.ERROR(line) - #if len(stderrlines) > 50: - # log.ERROR('...') raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): From agaynor at codespeak.net Sun Apr 4 19:43:45 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 19:43:45 +0200 (CEST) Subject: [pypy-svn] r73374 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404174345.3CAA5282C07@codespeak.net> Author: agaynor Date: Sun Apr 4 19:43:43 2010 New Revision: 73374 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: Implemented PyLong_{Check, CheckExact}. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Sun Apr 4 19:43:43 2010 @@ -1,9 +1,11 @@ from pypy.rpython.lltypesystem import lltype, rffi -from pypy.module.cpyext.api import cpython_api, PyObject +from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers from pypy.objspace.std.longobject import W_LongObject from pypy.interpreter.error import OperationError +PyLong_Check, PyLong_CheckExact = build_type_checkers("Long") + @cpython_api([lltype.Signed], PyObject) def PyLong_FromLong(space, val): """Return a new PyLongObject object from v, or NULL on failure.""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Sun Apr 4 19:43:43 2010 @@ -15,8 +15,27 @@ value = api.PyLong_FromLong(sys.maxint + 1) assert isinstance(value, W_LongObject) assert space.unwrap(value) == sys.maxint + 1 # should obviously fail but doesnt + def test_asulong(self, space, api): w_value = api.PyLong_FromLong((sys.maxint - 1) / 2) w_value = space.mul(w_value, space.wrap(4)) value = api.PyLong_AsUnsignedLong(w_value) assert value == (sys.maxint - 1) * 2 + + def test_type_check(self, space, api): + w_l = space.wrap(sys.maxint + 1) + assert api.PyLong_Check(w_l) + assert api.PyLong_CheckExact(w_l) + + w_i = space.wrap(sys.maxint) + assert not api.PyLong_Check(w_i) + assert not api.PyLong_CheckExact(w_i) + + L = space.appexec([], """(): + class L(long): + pass + return L + """) + l = space.call_function(L) + assert api.PyLong_Check(l) + assert not api.PyLong_CheckExact(l) From xoraxax at codespeak.net Sun Apr 4 19:52:46 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:52:46 +0200 (CEST) Subject: [pypy-svn] r73375 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404175246.5B237282C08@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:52:44 2010 New Revision: 73375 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h Log: Add macro and a define for PyTuple_GET_ITEM. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 4 19:52:44 2010 @@ -9,6 +9,7 @@ #define PY_LONG_LONG long long #define SIZEOF_LONG_LONG sizeof(PY_LONG_LONG) #define PY_FORMAT_SIZE_T "z" +#define SIZEOF_VOID_P sizeof(void *) #define WITH_DOC_STRINGS /* Compat stuff */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 4 19:52:44 2010 @@ -396,6 +396,14 @@ #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) +#define _PyObject_VAR_SIZE(typeobj, nitems) \ + (size_t) \ + ( ( (typeobj)->tp_basicsize + \ + (nitems)*(typeobj)->tp_itemsize + \ + (SIZEOF_VOID_P - 1) \ + ) & ~(SIZEOF_VOID_P - 1) \ + ) + #define PyObject_INIT(op, typeobj) \ ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) #define PyObject_INIT_VAR(op, typeobj, size) \ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/tupleobject.h Sun Apr 4 19:52:44 2010 @@ -11,6 +11,8 @@ PyObject * PyTuple_Pack(Py_ssize_t, ...); #define PyTuple_SET_ITEM PyTuple_SetItem +#define PyTuple_GET_ITEM PyTuple_GetItem + #ifdef __cplusplus } From xoraxax at codespeak.net Sun Apr 4 19:53:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:53:05 +0200 (CEST) Subject: [pypy-svn] r73376 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404175305.39FE2282C09@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:53:03 2010 New Revision: 73376 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Log: Add PySequence_GetItem. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Sun Apr 4 19:53:03 2010 @@ -48,7 +48,14 @@ def PySequence_GetSlice(space, w_obj, start, end): """Return the slice of sequence object o between i1 and i2, or NULL on failure. This is the equivalent of the Python expression o[i1:i2]. - + This function used an int type for i1 and i2. This might require changes in your code for properly supporting 64-bit systems.""" return space.getslice(w_obj, space.wrap(start), space.wrap(end)) + + at cpython_api([PyObject, Py_ssize_t], PyObject) +def PySequence_GetItem(space, w_obj, i): + """Return the ith element of o, or NULL on failure. This is the equivalent of + the Python expression o[i].""" + return space.getitem(w_obj, space.wrap(i)) + From xoraxax at codespeak.net Sun Apr 4 19:53:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 19:53:25 +0200 (CEST) Subject: [pypy-svn] r73377 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404175325.5599D282C0A@codespeak.net> Author: xoraxax Date: Sun Apr 4 19:53:23 2010 New Revision: 73377 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Log: Add PyInt_AsSsize_t PyInt_FromSsize_t. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Sun Apr 4 19:53:23 2010 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers) + build_type_checkers, Py_ssize_t) PyInt_Check, PyInt_CheckExact = build_type_checkers("Int") @@ -20,3 +20,20 @@ returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" return space.int_w(w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyInt_AsSsize_t(space, w_obj): + """Will first attempt to cast the object to a PyIntObject or + PyLongObject, if it is not already one, and then return its value as + Py_ssize_t. + """ + return space.int_w(w_obj) # XXX this is wrong on win64 + + at cpython_api([Py_ssize_t], PyObject) +def PyInt_FromSsize_t(space, ival): + """Create a new integer object with a value of ival. If the value is larger + than LONG_MAX or smaller than LONG_MIN, a long integer object is + returned. + """ + return space.wrap(ival) # XXX this is wrong on win64 + From agaynor at codespeak.net Sun Apr 4 20:18:37 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 20:18:37 +0200 (CEST) Subject: [pypy-svn] r73378 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404181837.3C13C282B9E@codespeak.net> Author: agaynor Date: Sun Apr 4 20:18:35 2010 New Revision: 73378 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Implemented PyObject_GetAttrString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 4 20:18:35 2010 @@ -9,6 +9,7 @@ from pypy.objspace.std.typeobject import W_TypeObject import pypy.module.__builtin__.operation as operation + @cpython_api([PyObject], PyObject) def _PyObject_New(space, w_type): if isinstance(w_type, W_PyCTypeObject): @@ -34,6 +35,14 @@ def PyObject_Not(space, w_obj): return not space.is_true(w_obj) + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyObject_GetAttrString(space, w_obj, name_ptr): + """Retrieve an attribute named attr_name from object o. Returns the attribute + value on success, or NULL on failure. This is the equivalent of the Python + expression o.attr_name.""" + name = rffi.charp2str(name_ptr) + return space.getattr(w_obj, space.wrap(name)) + @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_HasAttr(space, w_obj, w_name): w_res = operation.hasattr(space, w_obj, w_name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 20:18:35 2010 @@ -4325,13 +4325,6 @@ expression o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyObject_GetAttrString(space, o, attr_name): - """Retrieve an attribute named attr_name from object o. Returns the attribute - value on success, or NULL on failure. This is the equivalent of the Python - expression o.attr_name.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PyObject_GenericGetAttr(space, o, name): """Generic attribute getter function that is meant to be put into a type Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sun Apr 4 20:18:35 2010 @@ -1,6 +1,8 @@ import py from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.rpython.lltypesystem import rffi + class TestObject(BaseApiTest): def test_IsTrue(self, space, api): @@ -41,6 +43,13 @@ api.PyObject_SetAttr(space.wrap(x), space.wrap('test'), space.wrap(10)) assert x.test == 10 + def test_getattr_string(self, space, api): + assert api.PyObject_GetAttrString(space.wrap(""), rffi.str2charp("__len__")) + assert not api.PyObject_GetAttrString(space.wrap(""), rffi.str2charp("not_real")) + assert api.PyErr_Occurred() is space.w_AttributeError + api.PyErr_Clear() + + def test_getitem(self, space, api): w_t = space.wrap((1, 2, 3, 4, 5)) assert space.unwrap(api.PyObject_GetItem(w_t, space.wrap(3))) == 4 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 20:18:35 2010 @@ -83,6 +83,6 @@ assert fuu2(u"abc").baz().escape() def test_sre(self): - skip("In Progress") +# skip("In Progress") module = self.import_module(name='_sre') From fijal at codespeak.net Sun Apr 4 20:18:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 4 Apr 2010 20:18:43 +0200 (CEST) Subject: [pypy-svn] r73379 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404181843.1A6EB282C0B@codespeak.net> Author: fijal Date: Sun Apr 4 20:18:41 2010 New Revision: 73379 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: test and fix _PyString_Resize Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sun Apr 4 20:18:41 2010 @@ -8,6 +8,17 @@ PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") +def new_empty_str(space, length): + py_str = lltype.malloc(PyStringObject.TO, flavor='raw') + py_str.c_ob_refcnt = 1 + + buflen = length + 1 + py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') + py_str.c_buffer[buflen-1] = '\0' + py_str.c_size = length + py_str.c_ob_type = make_ref(space, space.w_str) + return py_str + @cpython_api_c() def PyString_FromFormatV(): pass @@ -19,16 +30,7 @@ ptr = make_ref(space, space.wrap(s)) return rffi.cast(PyStringObject, ptr) else: - py_str = lltype.malloc(PyStringObject.TO, flavor='raw') - py_str.c_ob_refcnt = 1 - - buflen = length + 1 - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - py_str.c_buffer[buflen-1] = '\0' - py_str.c_size = length - py_str.c_ob_type = make_ref(space, space.w_str) - - return py_str + return new_empty_str(space, length) @cpython_api([rffi.CCHARP], PyObject) def PyString_FromString(space, char_p): @@ -69,13 +71,23 @@ This function used an int type for newsize. This might require changes in your code for properly supporting 64-bit systems.""" # XXX always create a new string so far - w_s = from_ref(space, ref[0]) + py_str = rffi.cast(PyStringObject, ref[0]) + if not py_str.c_buffer: + raise OperationError(space.w_SystemError, space.wrap( + "_PyString_Resize called on already created string")) try: - ptr = make_ref(space, space.wrap(space.str_w(w_s)[:newsize])) - except: + py_newstr = new_empty_str(space, newsize) + except MemoryError: Py_DecRef(space, ref[0]) - ref[0] = lltype.nullptr(PyStringObject) + ref[0] = lltype.nullptr(PyObject) raise + to_cp = newsize + oldsize = py_str.c_size + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newstr.c_buffer[i] = py_str.c_buffer[i] + py_newstr.c_buffer[newsize] = '\x00' Py_DecRef(space, ref[0]) - ref[0] = ptr + ref[0] = rffi.cast(PyObject, py_newstr) return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Sun Apr 4 20:18:41 2010 @@ -1,15 +1,12 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.stringobject import new_empty_str +from pypy.module.cpyext.api import PyStringObject, PyObjectP, PyObject import py import sys -#class TestObject(BaseApiTest): -# def test_Size(self, space, api): -# s = space.wrap("test") -# assert api.PyString_Size(s) == 4 - class AppTestStringObject(AppTestCpythonExtensionBase): def test_stringobject(self): module = self.import_extension('foo', [ @@ -128,3 +125,26 @@ ''') res = module.test_string_format_v(1, "xyz") print res + +class TestString(BaseApiTest): + def test_string_resize(self, space, api): + py_str = new_empty_str(space, 10) + ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + py_str.c_buffer[0] = 'a' + py_str.c_buffer[1] = 'b' + py_str.c_buffer[2] = 'c' + ar[0] = rffi.cast(PyObject, py_str) + api._PyString_Resize(ar, 3) + py_str = rffi.cast(PyStringObject, ar[0]) + assert py_str.c_size == 3 + assert py_str.c_buffer[1] == 'b' + assert py_str.c_buffer[3] == '\x00' + # the same for growing + ar[0] = rffi.cast(PyObject, py_str) + api._PyString_Resize(ar, 10) + py_str = rffi.cast(PyStringObject, ar[0]) + assert py_str.c_size == 10 + assert py_str.c_buffer[1] == 'b' + assert py_str.c_buffer[10] == '\x00' + lltype.free(ar, flavor='raw') + From agaynor at codespeak.net Sun Apr 4 20:19:01 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 20:19:01 +0200 (CEST) Subject: [pypy-svn] r73380 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100404181901.8CAD8282C0C@codespeak.net> Author: agaynor Date: Sun Apr 4 20:19:00 2010 New Revision: 73380 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Oh shoot, re-comment out this test again. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Sun Apr 4 20:19:00 2010 @@ -83,6 +83,6 @@ assert fuu2(u"abc").baz().escape() def test_sre(self): -# skip("In Progress") + skip("In Progress") module = self.import_module(name='_sre') From xoraxax at codespeak.net Sun Apr 4 20:34:01 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 20:34:01 +0200 (CEST) Subject: [pypy-svn] r73381 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404183401.8B637282C0D@codespeak.net> Author: xoraxax Date: Sun Apr 4 20:34:00 2010 New Revision: 73381 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add PyVarObject and render all exception classes (!) as globals. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sun Apr 4 20:34:00 2010 @@ -21,12 +21,16 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize from pypy.rlib.exports import export_struct +from pypy.module import exceptions +from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException DEBUG_WRAPPER = False +# update these for other platforms Py_ssize_t = lltype.Signed +size_t = rffi.ULONG ADDR = lltype.Signed include_dir = py.path.local(autopath.pypydir) / 'module' / 'cpyext' / 'include' @@ -229,9 +233,8 @@ 'Py_False': ('PyObject*', 'space.w_False'), } -for exc_name in ['TypeError', 'ValueError', 'KeyError', 'Exception', - 'BaseException', 'SystemError', 'OSError']: - GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.w_' + exc_name) +for exc_name in exceptions.Module.interpleveldefs.keys(): + GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) for cpyname, pypyexpr in {"Type": "space.w_type", "BaseObject": "space.w_object", @@ -254,6 +257,8 @@ PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyObject)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('struct _object', PyObjectFields, PyObjectStruct) +PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) +PyVarObject = lltype.Ptr(PyVarObjectStruct) # a pointer to PyObject PyObjectP = rffi.CArrayPtr(PyObject) From xoraxax at codespeak.net Sun Apr 4 20:34:37 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 20:34:37 +0200 (CEST) Subject: [pypy-svn] r73382 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404183437.320D2282C0E@codespeak.net> Author: xoraxax Date: Sun Apr 4 20:34:35 2010 New Revision: 73382 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Implement PyObject_InitVar PyObject_Init Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 4 20:34:35 2010 @@ -1,10 +1,11 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - Py_ssize_t + Py_ssize_t, PyVarObject from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject +from pypy.module.cpyext.pyerrors import PyErr_NoMemory from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject import pypy.module.__builtin__.operation as operation @@ -72,3 +73,28 @@ """Return element of o corresponding to the object key or NULL on failure. This is the equivalent of the Python expression o[key].""" return space.getitem(w_obj, w_key) + + at cpython_api([PyObject, PyTypeObjectPtr], PyObject, borrowed=True) +def PyObject_Init(space, op, type): + """Initialize a newly-allocated object op with its type and initial + reference. Returns the initialized object. If type indicates that the + object participates in the cyclic garbage detector, it is added to the + detector's set of observed objects. Other fields of the object are not + affected.""" + if not op: + PyErr_NoMemory(space) + op.c_ob_type = type + op.c_ob_refcnt = 1 + return op + + at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject, borrowed=True) +def PyObject_InitVar(space, op, type, size): + """This does everything PyObject_Init() does, and also initializes the + length information for a variable-size object.""" + if not op: + PyErr_NoMemory(space) + op.c_ob_size = size + op.c_ob_type = type + op.c_ob_refcnt = 1 + return rffi.cast(PyObject, op) + From xoraxax at codespeak.net Sun Apr 4 20:35:03 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 20:35:03 +0200 (CEST) Subject: [pypy-svn] r73383 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404183503.4807D282C0F@codespeak.net> Author: xoraxax Date: Sun Apr 4 20:35:01 2010 New Revision: 73383 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Log: Add comments and a statichere define. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 4 20:35:01 2010 @@ -39,6 +39,8 @@ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) #endif +#define statichere static + #define Py_MEMCPY memcpy #include Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Sun Apr 4 20:35:01 2010 @@ -5,8 +5,8 @@ #define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) #define PyMem_FREE free -/* XXX use obmalloc like cpython does */ +/* XXX use obmalloc like cpython and pypy do, otherwise we might get segfaults */ #define PyObject_MALLOC PyMem_MALLOC -#define PyObject_REALLOC PyMem_REALLOC +//#define PyObject_REALLOC PyMem_REALLOC #define PyObject_FREE PyMem_FREE From xoraxax at codespeak.net Sun Apr 4 20:38:31 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 20:38:31 +0200 (CEST) Subject: [pypy-svn] r73384 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100404183831.99411282C10@codespeak.net> Author: xoraxax Date: Sun Apr 4 20:38:30 2010 New Revision: 73384 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: Add macro and fix signature. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 4 20:38:30 2010 @@ -396,6 +396,7 @@ #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) +#define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize ) #define _PyObject_VAR_SIZE(typeobj, nitems) \ (size_t) \ ( ( (typeobj)->tp_basicsize + \ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Sun Apr 4 20:38:30 2010 @@ -170,7 +170,7 @@ return space.wrap(W_PyCWrapperObject(space, pto, method_name, wrapper_func, doc, flags, func)) - at cpython_api([PyMethodDef, PyObject, rffi.CCHARP], PyObject) + at cpython_api([lltype.Ptr(PyMethodDef), PyObject, rffi.CCHARP], PyObject) def Py_FindMethod(space, table, w_ob, name_ptr): """Return a bound method object for an extension type implemented in C. This can be useful in the implementation of a tp_getattro or From xoraxax at codespeak.net Sun Apr 4 20:52:59 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 20:52:59 +0200 (CEST) Subject: [pypy-svn] r73385 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404185259.812A4282C11@codespeak.net> Author: xoraxax Date: Sun Apr 4 20:52:57 2010 New Revision: 73385 Added: pypy/branch/cpython-extension/pypy/module/cpyext/iterator.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Add PyCallIter_New. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Sun Apr 4 20:52:57 2010 @@ -48,6 +48,7 @@ import pypy.module.cpyext.eval import pypy.module.cpyext.getargs import pypy.module.cpyext.import_ +import pypy.module.cpyext.iterator # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/iterator.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/iterator.py Sun Apr 4 20:52:57 2010 @@ -0,0 +1,14 @@ +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject +from pypy.interpreter.error import OperationError +import pypy.module.__builtin__.operation as operation + + + at cpython_api([PyObject, PyObject], PyObject) +def PyCallIter_New(space, w_callable, w_sentinel): + """Return a new iterator. The first parameter, callable, can be any Python + callable object that can be called with no parameters; each call to it should + return the next item in the iteration. When callable returns a value equal to + sentinel, the iteration will be terminated. + """ + return operation.iter_sentinel(space, w_callable, w_sentinel) + From agaynor at codespeak.net Sun Apr 4 20:55:42 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 4 Apr 2010 20:55:42 +0200 (CEST) Subject: [pypy-svn] r73386 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404185542.35BBA282C12@codespeak.net> Author: agaynor Date: Sun Apr 4 20:55:40 2010 New Revision: 73386 Added: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Implemented PyMapping_Keys. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Sun Apr 4 20:55:40 2010 @@ -48,6 +48,7 @@ import pypy.module.cpyext.eval import pypy.module.cpyext.getargs import pypy.module.cpyext.import_ +import pypy.module.cpyext.mapping import pypy.module.cpyext.iterator # now that all rffi_platform.Struct types are registered, configure them Added: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Sun Apr 4 20:55:40 2010 @@ -0,0 +1,12 @@ +from pypy.module.cpyext.api import cpython_api +from pypy.module.cpyext.pyobject import PyObject + + + at cpython_api([PyObject], PyObject) +def PyMapping_Keys(space, w_obj): + """On success, return a list of the keys in object o. On failure, return NULL. + This is equivalent to the Python expression o.keys().""" + # XXX: Cpython implements this in terms of PyObject_CallMethod, we should + # do that eventually. + w_meth = space.getattr(w_obj, space.wrap("keys")) + return space.call_function(w_meth) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 4 20:55:40 2010 @@ -3674,12 +3674,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyMapping_Keys(space, o): - """On success, return a list of the keys in object o. On failure, return NULL. - This is equivalent to the Python expression o.keys().""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyMapping_Values(space, o): """On success, return a list of the values in object o. On failure, return NULL. This is equivalent to the Python expression o.values().""" Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Sun Apr 4 20:55:40 2010 @@ -0,0 +1,8 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest + + +class TestMapping(BaseApiTest): + def test_keys(self, space, api): + w_d = space.newdict() + space.setitem(w_d, space.wrap("a"), space.wrap("a")) + assert space.eq_w(api.PyMapping_Keys(w_d), space.wrap(["a"])) From xoraxax at codespeak.net Sun Apr 4 21:11:38 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 4 Apr 2010 21:11:38 +0200 (CEST) Subject: [pypy-svn] r73387 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100404191138.9C176282C13@codespeak.net> Author: xoraxax Date: Sun Apr 4 21:11:12 2010 New Revision: 73387 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Log: Enable HAVE_UNICODE and add typedefs and defines. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 4 21:11:12 2010 @@ -11,6 +11,7 @@ #define PY_FORMAT_SIZE_T "z" #define SIZEOF_VOID_P sizeof(void *) #define WITH_DOC_STRINGS +#define HAVE_UNICODE /* Compat stuff */ #ifndef _WIN32 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Sun Apr 4 21:11:12 2010 @@ -1,3 +1,9 @@ typedef struct { PyObject_HEAD } PyUnicodeObject; +//XXX +typedef unsigned int Py_UCS4; +// pypy only supports only UCS4 +#define PY_UNICODE_TYPE Py_UCS4 +typedef PY_UNICODE_TYPE Py_UNICODE; + From trundle at codespeak.net Mon Apr 5 01:08:53 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 01:08:53 +0200 (CEST) Subject: [pypy-svn] r73388 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100404230853.434B3282B90@codespeak.net> Author: trundle Date: Mon Apr 5 01:08:51 2010 New Revision: 73388 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Add various Py_UNICODE_IS* functions and Py_UNICODE_TOLOWER. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Mon Apr 5 01:08:51 2010 @@ -50,6 +50,7 @@ import pypy.module.cpyext.import_ import pypy.module.cpyext.mapping import pypy.module.cpyext.iterator +import pypy.module.cpyext.unicodeobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 5 01:08:51 2010 @@ -5690,36 +5690,11 @@ raise NotImplementedError @cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISSPACE(space, ch): - """Return 1 or 0 depending on whether ch is a whitespace character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISLOWER(space, ch): - """Return 1 or 0 depending on whether ch is a lowercase character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISUPPER(space, ch): - """Return 1 or 0 depending on whether ch is an uppercase character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], rffi.INT_real) def Py_UNICODE_ISTITLE(space, ch): """Return 1 or 0 depending on whether ch is a titlecase character.""" raise NotImplementedError @cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISLINEBREAK(space, ch): - """Return 1 or 0 depending on whether ch is a linebreak character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISDECIMAL(space, ch): - """Return 1 or 0 depending on whether ch is a decimal character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], rffi.INT_real) def Py_UNICODE_ISDIGIT(space, ch): """Return 1 or 0 depending on whether ch is a digit character.""" raise NotImplementedError @@ -5734,16 +5709,6 @@ """Return 1 or 0 depending on whether ch is an alphabetic character.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) -def Py_UNICODE_ISALNUM(space, ch): - """Return 1 or 0 depending on whether ch is an alphanumeric character.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], {Py_UNICODE}) -def Py_UNICODE_TOLOWER(space, ch): - """Return the character ch converted to lower case.""" - raise NotImplementedError - @cpython_api([{Py_UNICODE}], {Py_UNICODE}) def Py_UNICODE_TOUPPER(space, ch): """Return the character ch converted to upper case.""" Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Mon Apr 5 01:08:51 2010 @@ -0,0 +1,31 @@ +# encoding: iso-8859-15 +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestUnicode(BaseApiTest): + def test_IS(self, space, api): + for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, + 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, + 0x2009, 0x200a, + #0x200b is in Other_Default_Ignorable_Code_Point in 4.1.0 + 0x2028, 0x2029, 0x202f, 0x205f, 0x3000]: + assert api.Py_UNICODE_ISSPACE(char) + assert not api.Py_UNICODE_ISSPACE(ord(u'a')) + + assert api.Py_UNICODE_ISDECIMAL(ord(u'\u0660')) + assert not api.Py_UNICODE_ISDECIMAL(ord(u'a')) + + for char in [0x0a, 0x0d, 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029]: + assert api.Py_UNICODE_ISLINEBREAK(char) + + assert api.Py_UNICODE_ISLOWER(ord(u'?')) + assert not api.Py_UNICODE_ISUPPER(ord(u'?')) + assert api.Py_UNICODE_ISLOWER(ord(u'a')) + assert not api.Py_UNICODE_ISUPPER(ord(u'a')) + assert not api.Py_UNICODE_ISLOWER(ord(u'?')) + assert api.Py_UNICODE_ISUPPER(ord(u'?')) + + def test_TOLOWER(self, space, api): + assert api.Py_UNICODE_TOLOWER(ord(u'?')) == ord(u'?') + assert api.Py_UNICODE_TOLOWER(ord(u'?')) == ord(u'?') + Added: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Mon Apr 5 01:08:51 2010 @@ -0,0 +1,53 @@ +from pypy.rpython.lltypesystem import rffi +from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb +from pypy.module.cpyext.api import CANNOT_FAIL, build_type_checkers, cpython_api + +PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode") + +# XXX +Py_UNICODE = rffi.UINT + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISSPACE(space, w_ch): + """Return 1 or 0 depending on whether ch is a whitespace character.""" + return unicodedb.isspace(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISALNUM(space, w_ch): + """Return 1 or 0 depending on whether ch is an alphanumeric character.""" + return unicodedb.isalnum(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISLINEBREAK(space, w_ch): + """Return 1 or 0 depending on whether ch is a linebreak character.""" + return unicodedb.islinebreak(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISSPACE(space, w_ch): + """Return 1 or 0 depending on whether ch is a whitespace character.""" + return unicodedb.isspace(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISALNUM(space, w_ch): + """Return 1 or 0 depending on whether ch is an alphanumeric character.""" + return unicodedb.isalnum(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISDECIMAL(space, w_ch): + """Return 1 or 0 depending on whether ch is a decimal character.""" + return unicodedb.isdecimal(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISLOWER(space, w_ch): + """Return 1 or 0 depending on whether ch is a lowercase character.""" + return unicodedb.islower(w_ch) + + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) +def Py_UNICODE_ISUPPER(space, w_ch): + """Return 1 or 0 depending on whether ch is an uppercase character.""" + return unicodedb.isupper(w_ch) + + at cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL) +def Py_UNICODE_TOLOWER(space, w_ch): + """Return the character ch converted to lower case.""" + return unicodedb.tolower(w_ch) From antoine at codespeak.net Mon Apr 5 01:12:12 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Mon, 5 Apr 2010 01:12:12 +0200 (CEST) Subject: [pypy-svn] r73389 - pypy/branch/decouple-host-opcodes Message-ID: <20100404231212.AB9ED282B90@codespeak.net> Author: antoine Date: Mon Apr 5 01:12:11 2010 New Revision: 73389 Added: pypy/branch/decouple-host-opcodes/ (props changed) - copied from r73388, pypy/trunk/ Log: A branch to try and decouple host opcodes from pypy opcodes From trundle at codespeak.net Mon Apr 5 01:52:27 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 01:52:27 +0200 (CEST) Subject: [pypy-svn] r73390 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100404235227.46C9B282B90@codespeak.net> Author: trundle Date: Mon Apr 5 01:52:25 2010 New Revision: 73390 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: Add missing import. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Mon Apr 5 01:52:25 2010 @@ -1,3 +1,4 @@ +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, From trundle at codespeak.net Mon Apr 5 02:01:55 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 02:01:55 +0200 (CEST) Subject: [pypy-svn] r73391 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405000155.41D0C282B90@codespeak.net> Author: trundle Date: Mon Apr 5 02:01:53 2010 New Revision: 73391 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: Typo. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Mon Apr 5 02:01:53 2010 @@ -179,7 +179,7 @@ # XXX handle __doc__ name = rffi.charp2str(name_ptr) - methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) + methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), table) if methods: i = -1 while True: From xoraxax at codespeak.net Mon Apr 5 02:02:45 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 02:02:45 +0200 (CEST) Subject: [pypy-svn] r73392 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405000245.E9F4F282B90@codespeak.net> Author: xoraxax Date: Mon Apr 5 02:02:44 2010 New Revision: 73392 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Cast exceptions correctly and also include bool and none types. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 02:02:44 2010 @@ -234,7 +234,7 @@ } for exc_name in exceptions.Module.interpleveldefs.keys(): - GLOBALS['PyExc_' + exc_name] = ('PyObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) + GLOBALS['PyExc_' + exc_name] = ('PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) for cpyname, pypyexpr in {"Type": "space.w_type", "BaseObject": "space.w_object", @@ -242,6 +242,8 @@ "Tuple": "space.w_tuple", "List": "space.w_list", "Unicode": "space.w_unicode", + 'Bool': 'space.w_bool', + 'None': 'space.w_None', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From trundle at codespeak.net Mon Apr 5 02:07:00 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 02:07:00 +0200 (CEST) Subject: [pypy-svn] r73393 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405000700.44C5E282B90@codespeak.net> Author: trundle Date: Mon Apr 5 02:06:58 2010 New Revision: 73393 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Log: Add missing import. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Mon Apr 5 02:06:58 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, register_container from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError From antoine at codespeak.net Mon Apr 5 02:16:11 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Mon, 5 Apr 2010 02:16:11 +0200 (CEST) Subject: [pypy-svn] r73394 - in pypy/branch/decouple-host-opcodes/pypy: . interpreter objspace/flow objspace/flow/test objspace/std tool Message-ID: <20100405001611.2752D282B90@codespeak.net> Author: antoine Date: Mon Apr 5 02:16:09 2010 New Revision: 73394 Modified: pypy/branch/decouple-host-opcodes/pypy/ (props changed) pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py Log: Initial draft, using separate subclasses for pypy bytecode frames and host bytecode frames. Not sure it translates properly, although I've already fixed a couple of issues. Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Mon Apr 5 02:16:09 2010 @@ -257,8 +257,13 @@ self.actionflag.register_action(self.user_del_action) self.actionflag.register_action(self.frame_trace_action) - from pypy.interpreter.pyframe import PyFrame - self.FrameClass = PyFrame # can be overridden to a subclass + from pypy.interpreter.pycode import cpython_magic, default_magic + from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame + self.our_magic = default_magic + self.host_magic = cpython_magic + # can be overridden to a subclass + self.FrameClass = PyPyFrame + self.HostFrameClass = HostPyFrame if self.config.objspace.logbytecodes: self.bytecodecounts = [0] * 256 @@ -589,7 +594,12 @@ def createframe(self, code, w_globals, closure=None): "Create an empty PyFrame suitable for this code object." - return self.FrameClass(self, code, w_globals, closure) + magic = code.magic + if magic == self.host_magic: + return self.HostFrameClass(self, code, w_globals, closure) + elif magic == self.our_magic: + return self.FrameClass(self, code, w_globals, closure) + raise ValueError("bad magic %s" % magic) def allocate_lock(self): """Return an interp-level Lock object if threads are enabled, Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py Mon Apr 5 02:16:09 2010 @@ -236,9 +236,9 @@ from pypy.interpreter.gateway import BuiltinCode if isinstance(self.code, BuiltinCode): identifier = self.code.identifier - if Function._all.get(identifier, self) is not self: - print "builtin code identifier %s used twice: %s and %s" % ( - identifier, self, Function._all[identifier]) + #if Function._all.get(identifier, self) is not self: + #print "builtin code identifier %s used twice: %s and %s" % ( + #identifier, self, Function._all[identifier]) # we have been seen by other means so rtyping should not choke # on us Function._all[identifier] = self Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py Mon Apr 5 02:16:09 2010 @@ -135,8 +135,9 @@ from pypy.rlib import rstack # the following 'assert' is an annotation hint: it hides from # the annotator all methods that are defined in PyFrame but - # overridden in the FrameClass subclass of PyFrame. - assert isinstance(self, self.space.FrameClass) + # overridden in the {,Host}FrameClass subclasses of PyFrame. + assert (isinstance(self, self.space.FrameClass) + or isinstance(self, self.space.HostFrameClass)) executioncontext = self.space.getexecutioncontext() executioncontext.enter(self) try: @@ -628,6 +629,13 @@ return space.wrap(self.builtin is not space.builtin) return space.w_False + +class PyPyFrame(PyFrame): + pass + +class HostPyFrame(PyFrame): + pass + # ____________________________________________________________ def get_block_class(opname): Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Mon Apr 5 02:16:09 2010 @@ -14,9 +14,9 @@ from pypy.rlib import jit, rstackovf from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.tool.stdlib_opcode import (opcodedesc, HAVE_ARGUMENT, - unrolling_opcode_descs, - opcode_method_names) +from pypy.tool.stdlib_opcode import (unrolling_opcode_descs, + HAVE_ARGUMENT, host_HAVE_ARGUMENT, opcodedesc, host_opcodedesc, + opcode_method_names, host_opcode_method_names, ) def unaryoperation(operationname): """NOT_RPYTHON""" @@ -180,7 +180,7 @@ probs[opcode] = probs.get(opcode, 0) + 1 self.last_opcode = opcode - if opcode >= HAVE_ARGUMENT: + if opcode >= self.HAVE_ARGUMENT: lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 @@ -188,16 +188,16 @@ else: oparg = 0 - while opcode == opcodedesc.EXTENDED_ARG.index: + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) - if opcode < HAVE_ARGUMENT: + if opcode < self.HAVE_ARGUMENT: raise BytecodeCorruption lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 oparg = (oparg << 16) | (hi << 8) | lo - if opcode == opcodedesc.RETURN_VALUE.index: + if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() block = self.unrollstack(SReturnValue.kind) if block is None: @@ -208,11 +208,11 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == opcodedesc.YIELD_VALUE.index: + if opcode == self.opcodedesc.YIELD_VALUE.index: #self.last_instr = intmask(next_instr - 1) XXX clean up! raise Yield - if opcode == opcodedesc.END_FINALLY.index: + if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): # go on unrolling the stack @@ -225,7 +225,7 @@ next_instr = block.handle(self, unroller) return next_instr - if opcode == opcodedesc.JUMP_ABSOLUTE.index: + if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, next_instr, ec) if we_are_translated(): @@ -242,7 +242,7 @@ # dispatch to the opcode method meth = getattr(self, opdesc.methodname) res = meth(oparg, next_instr) - if opdesc.index == opcodedesc.CALL_FUNCTION.index: + if opdesc.index == self.opcodedesc.CALL_FUNCTION.index: rstack.resume_point("dispatch_call", self, co_code, next_instr, ec) # !! warning, for the annotator the next line is not # comparing an int and None - you can't do that. @@ -254,7 +254,7 @@ self.MISSING_OPCODE(oparg, next_instr) else: # when we are not translated, a list lookup is much faster - methodname = opcode_method_names[opcode] + methodname = self.opcode_method_names[opcode] res = getattr(self, methodname)(oparg, next_instr) if res is not None: next_instr = res @@ -690,26 +690,6 @@ w_list = self.space.newlist(items) self.pushvalue(w_list) - def BUILD_MAP(self, itemcount, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - # We could pre-allocate a dict here - # but for the moment this code is not translated. - pass - else: - if itemcount != 0: - raise BytecodeCorruption - w_dict = self.space.newdict() - self.pushvalue(w_dict) - - def STORE_MAP(self, zero, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - w_key = self.popvalue() - w_value = self.popvalue() - w_dict = self.peekvalue() - self.space.setitem(w_dict, w_key, w_value) - else: - raise BytecodeCorruption - def LOAD_ATTR(self, nameindex, next_instr): "obj.attributename" w_attributename = self.getname_w(nameindex) @@ -1020,6 +1000,48 @@ STOP_CODE = MISSING_OPCODE +class __extend__(pyframe.PyPyFrame): + opcode_method_names = opcode_method_names + opcodedesc = opcodedesc + HAVE_ARGUMENT = HAVE_ARGUMENT + + def BUILD_MAP(self, itemcount, next_instr): + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + + def STORE_MAP(self, zero, next_instr): + raise BytecodeCorruption + +host_version_info = sys.version_info + +class __extend__(pyframe.HostPyFrame): + opcode_method_names = host_opcode_method_names + opcodedesc = host_opcodedesc + HAVE_ARGUMENT = host_HAVE_ARGUMENT + + def BUILD_MAP(self, itemcount, next_instr): + if host_version_info >= (2, 6): + # We could pre-allocate a dict here + # but for the moment this code is not translated. + pass + else: + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + + def STORE_MAP(self, zero, next_instr): + if host_version_info >= (2, 6): + w_key = self.popvalue() + w_value = self.popvalue() + w_dict = self.peekvalue() + self.space.setitem(w_dict, w_key, w_value) + else: + raise BytecodeCorruption + + ### ____________________________________________________________ ### class ExitFrame(Exception): Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py Mon Apr 5 02:16:09 2010 @@ -1,7 +1,7 @@ import collections from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError -from pypy.interpreter import pyframe +from pypy.interpreter import pyframe, pycode from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState @@ -219,8 +219,12 @@ # create an empty frame suitable for the code object # while ignoring any operation like the creation of the locals dict self.recorder = [] - frame = FlowSpaceFrame(self.space, self.code, - self.w_globals, self.closure) + if self.code.magic == pycode.cpython_magic: + frame = FlowSpaceHostPyFrame(self.space, self.code, + self.w_globals, self.closure) + else: + frame = FlowSpacePyPyFrame(self.space, self.code, + self.w_globals, self.closure) frame.last_instr = 0 return frame @@ -398,7 +402,7 @@ if w_v.value is oldvalue: stack_items_w[i] = w_new -class FlowSpaceFrame(pyframe.PyFrame): +class FlowSpaceFrameBase(object): def make_arguments(self, nargs): return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) def argument_factory(self, *args): @@ -410,3 +414,10 @@ raise operr return pyframe.PyFrame.handle_operation_error(self, ec, operr, *args, **kwds) + +class FlowSpacePyPyFrame(FlowSpaceFrameBase, pyframe.PyPyFrame): + pass + +class FlowSpaceHostPyFrame(FlowSpaceFrameBase, pyframe.HostPyFrame): + pass + Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py Mon Apr 5 02:16:09 2010 @@ -826,23 +826,34 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - class X: - def m(self): - return 3 - - def f(): - x = X() - return x.m() - - # this code is generated by pypy-c when compiling above f - pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' - new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) - f2 = new.function(new_c, locals(), 'f') - - graph = self.codetest(f2) - all_ops = self.all_operations(graph) - assert all_ops['simple_call'] == 2 - assert all_ops['getattr'] == 1 + from pypy.objspace.flow import flowcontext + old_class = flowcontext.FlowSpaceHostPyFrame + try: + # HACK: we will replace a host-generated code object with a + # pypy-generated code object, but it will carry the host's + # magic number (since it's generated with the host's code.new). + flowcontext.FlowSpaceHostPyFrame = flowcontext.FlowSpacePyPyFrame + class X: + def m(self): + return 3 + + def f(): + x = X() + return x.m() + + # this code is generated by pypy-c when compiling above f + pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' + new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) + f2 = new.function(new_c, locals(), 'f') + import dis + dis.dis(f2) + + graph = self.codetest(f2) + all_ops = self.all_operations(graph) + assert all_ops['simple_call'] == 2 + assert all_ops['getattr'] == 1 + finally: + flowcontext.FlowSpaceHostPyFrame = old_class def test_generator(self): def f(): Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py Mon Apr 5 02:16:09 2010 @@ -3,14 +3,16 @@ import operator from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter import pyframe, pyopcode, function +from pypy.interpreter import pyopcode, function from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module.__builtin__ import OPTIMIZED_BUILTINS, Module from pypy.objspace.std.multimethod import FailedToImplement -class BaseFrame(pyframe.PyFrame): + +class BaseFrame(object): """These opcodes are always overridden.""" + _mixin_ = True def LIST_APPEND(f, oparg, next_instr): from pypy.objspace.std.listobject import W_ListObject @@ -135,9 +137,9 @@ f.pushvalue(w_result) -def build_frame(space): +def build_frame(space, baseclass): """Consider the objspace config and return a patched frame object.""" - class StdObjSpaceFrame(BaseFrame): + class StdObjSpaceFrame(baseclass, BaseFrame): pass if space.config.objspace.std.optimized_int_add: if space.config.objspace.std.withsmallint: Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py Mon Apr 5 02:16:09 2010 @@ -47,7 +47,9 @@ # setup all the object types and implementations self.model = model.StdTypeModel(self.config) - self.FrameClass = frame.build_frame(self) + from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame + self.FrameClass = frame.build_frame(self, PyPyFrame) + self.HostFrameClass = frame.build_frame(self, HostPyFrame) if self.config.objspace.std.withrope: self.StringObjectCls = W_RopeObject @@ -133,7 +135,7 @@ if not we_are_translated() and isinstance(code, CPythonFakeCode): return CPythonFakeFrame(self, code, w_globals) else: - return self.FrameClass(self, code, w_globals, closure) + return ObjSpace.createframe(self, code, w_globals, closure) def gettypefor(self, cls): return self.gettypeobject(cls.typedef) Modified: pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py Mon Apr 5 02:16:09 2010 @@ -4,6 +4,9 @@ 'hasconst', 'hasname', 'hasjrel', 'hasjabs', 'haslocal', 'hascompare', 'hasfree', 'cmp_op'] +from opcode import ( + opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) + def load_opcode(): import py opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') @@ -23,65 +26,83 @@ STORE_SLICE = opmap["STORE_SLICE+0"] DELETE_SLICE = opmap["DELETE_SLICE+0"] -opcode_method_names = ['MISSING_OPCODE'] * 256 -for name, index in opmap.items(): - opcode_method_names[index] = name.replace('+', '_') +def make_method_names(opmap): + tbl = ['MISSING_OPCODE'] * 256 + for name, index in opmap.items(): + tbl[index] = name.replace('+', '_') + return tbl + +opcode_method_names = make_method_names(opmap) +host_opcode_method_names = make_method_names(host_opmap) +#print ( + #set(enumerate(opcode_method_names)) ^ set(enumerate(host_opcode_method_names)) +#) +del make_method_names # ____________________________________________________________ # RPython-friendly helpers and structures from pypy.rlib.unroll import unrolling_iterable +def make_opcode_desc(HAVE_ARGUMENT): + class OpcodeDesc(object): + def __init__(self, name, index): + self.name = name + self.methodname = opcode_method_names[index] + self.index = index + self.hasarg = index >= HAVE_ARGUMENT + + def _freeze_(self): + return True + + def is_enabled(self, space): + """Check if the opcode should be enabled in the space's configuration. + (Returns True for all standard opcodes.)""" + opt = space.config.objspace.opcodes + return getattr(opt, self.name, True) + is_enabled._annspecialcase_ = 'specialize:memo' + + # for predictable results, we try to order opcodes most-used-first + opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] + + def sortkey(self): + try: + i = self.opcodeorder.index(self.index) + except ValueError: + i = 1000000 + return i, self.index + + def __cmp__(self, other): + return (cmp(self.__class__, other.__class__) or + cmp(self.sortkey(), other.sortkey())) -class OpcodeDesc(object): - def __init__(self, name, index): - self.name = name - self.methodname = opcode_method_names[index] - self.index = index - self.hasarg = index >= HAVE_ARGUMENT - - def _freeze_(self): - return True - - def is_enabled(self, space): - """Check if the opcode should be enabled in the space's configuration. - (Returns True for all standard opcodes.)""" - opt = space.config.objspace.opcodes - return getattr(opt, self.name, True) - is_enabled._annspecialcase_ = 'specialize:memo' - - # for predictable results, we try to order opcodes most-used-first - opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] - - def sortkey(self): - try: - i = self.opcodeorder.index(self.index) - except ValueError: - i = 1000000 - return i, self.index - - def __cmp__(self, other): - return (cmp(self.__class__, other.__class__) or - cmp(self.sortkey(), other.sortkey())) + return OpcodeDesc + +OpcodeDesc = make_opcode_desc(HAVE_ARGUMENT) +HostOpcodeDesc = make_opcode_desc(host_HAVE_ARGUMENT) opdescmap = {} -class opcodedesc: +class _baseopcodedesc: + pass + +class opcodedesc(_baseopcodedesc): """A namespace mapping OPCODE_NAME to OpcodeDescs.""" +class host_opcodedesc(_baseopcodedesc): + """A namespace mapping OPCODE_NAME to HostOpcodeDescs.""" + for name, index in opmap.items(): desc = OpcodeDesc(name, index) setattr(opcodedesc, name, desc) opdescmap[index] = desc +for name, index in host_opmap.items(): + desc = HostOpcodeDesc(name, index) + setattr(host_opcodedesc, name, desc) + lst = opdescmap.values() lst.sort() unrolling_opcode_descs = unrolling_iterable(lst) -# Allow non-translated code to interpret the new 2.6 bytecodes -import sys -if sys.version_info >= (2, 6): - import opcode - opcode_method_names[opcode.opmap['STORE_MAP']] = 'STORE_MAP' - del name, index, desc, lst From trundle at codespeak.net Mon Apr 5 02:22:57 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 02:22:57 +0200 (CEST) Subject: [pypy-svn] r73395 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100405002257.4BE89282B90@codespeak.net> Author: trundle Date: Mon Apr 5 02:22:55 2010 New Revision: 73395 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_GET_SIZE. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 5 02:22:55 2010 @@ -5654,15 +5654,6 @@ raise NotImplementedError @cpython_api([PyObject], Py_ssize_t) -def PyUnicode_GET_SIZE(space, o): - """Return the size of the object. o has to be a PyUnicodeObject (not - checked). - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t) def PyUnicode_GET_DATA_SIZE(space, o): """Return the size of the object's internal buffer in bytes. o has to be a PyUnicodeObject (not checked). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Mon Apr 5 02:22:55 2010 @@ -2,6 +2,9 @@ from pypy.module.cpyext.test.test_api import BaseApiTest class TestUnicode(BaseApiTest): + def test_unicodeobject(self, space, api): + assert space.unwrap(api.PyUnicode_GET_SIZE(space.wrap(u'sp?m'))) == 4 + def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Mon Apr 5 02:22:55 2010 @@ -1,6 +1,9 @@ from pypy.rpython.lltypesystem import rffi from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb -from pypy.module.cpyext.api import CANNOT_FAIL, build_type_checkers, cpython_api +from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, + build_type_checkers, cpython_api) +from pypy.module.cpyext.pyobject import PyObject +from pypy.objspace.std import unicodeobject PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode") @@ -51,3 +54,13 @@ def Py_UNICODE_TOLOWER(space, w_ch): """Return the character ch converted to lower case.""" return unicodedb.tolower(w_ch) + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyUnicode_GET_SIZE(space, w_obj): + """Return the size of the object. o has to be a PyUnicodeObject (not + checked). + + This function returned an int type. This might require changes + in your code for properly supporting 64-bit systems.""" + assert isinstance(w_obj, unicodeobject.W_UnicodeObject) + return space.len(w_obj) From antoine at codespeak.net Mon Apr 5 02:39:59 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Mon, 5 Apr 2010 02:39:59 +0200 (CEST) Subject: [pypy-svn] r73396 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100405003959.74FE6282B90@codespeak.net> Author: antoine Date: Mon Apr 5 02:39:58 2010 New Revision: 73396 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Log: Seems to help translation Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Mon Apr 5 02:39:58 2010 @@ -594,6 +594,8 @@ def createframe(self, code, w_globals, closure=None): "Create an empty PyFrame suitable for this code object." + from pypy.interpreter.pycode import PyCode + assert isinstance(code, PyCode) magic = code.magic if magic == self.host_magic: return self.HostFrameClass(self, code, w_globals, closure) From xoraxax at codespeak.net Mon Apr 5 02:47:16 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 02:47:16 +0200 (CEST) Subject: [pypy-svn] r73397 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405004716.F3A49282B90@codespeak.net> Author: xoraxax Date: Mon Apr 5 02:47:15 2010 New Revision: 73397 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Log: Let ob_type be a PyTypeObjectPtr. Fix translation in slotdefs.py. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 02:47:15 2010 @@ -251,12 +251,14 @@ from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] +PyTypeObject = lltype.ForwardReference() +PyTypeObjectPtr = lltype.Ptr(PyTypeObject) # It is important that these PyObjects are allocated in a raw fashion # Thus we cannot save a forward pointer to the wrapped object # So we need a forward and backward mapping in our State instance PyObjectStruct = lltype.ForwardReference() PyObject = lltype.Ptr(PyObjectStruct) -PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyObject)) +PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyTypeObjectPtr)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('struct _object', PyObjectFields, PyObjectStruct) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) @@ -387,8 +389,8 @@ object_pto = make_ref(space, space.w_object) object_pto = rffi.cast(PyTypeObjectPtr, object_pto) type_pto.c_tp_base = object_pto - type_pto.c_ob_type = make_ref(space, space.w_type) - object_pto.c_ob_type = make_ref(space, space.w_type) + type_pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_type)) + object_pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_type)) PyPyType_Ready(space, object_pto, space.w_object) PyPyType_Ready(space, type_pto, space.w_type) type_pto.c_tp_bases = make_ref(space, space.newtuple([space.w_object])) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Mon Apr 5 02:47:15 2010 @@ -85,7 +85,7 @@ PyErr_NoMemory(space) op.c_ob_type = type op.c_ob_refcnt = 1 - return op + return from_ref(space, op) # XXX will give an exception @cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject, borrowed=True) def PyObject_InitVar(space, op, type, size): @@ -96,5 +96,5 @@ op.c_ob_size = size op.c_ob_type = type op.c_ob_refcnt = 1 - return rffi.cast(PyObject, op) + return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Mon Apr 5 02:47:15 2010 @@ -63,7 +63,7 @@ flavor="raw", zero=True) py_obj = rffi.cast(PyObject, py_obj_pad) py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyObject, pto) + py_obj.c_ob_type = pto w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) elif isinstance(w_obj, W_StringObject): py_obj_str = lltype.malloc(PyStringObject.TO, flavor='raw', zero=True) @@ -72,12 +72,12 @@ pto = make_ref(space, space.w_str) py_obj = rffi.cast(PyObject, py_obj_str) py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyObject, pto) + py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) else: py_obj = lltype.malloc(PyObject.TO, flavor="raw", zero=True) py_obj.c_ob_refcnt = 1 pto = make_ref(space, space.type(w_obj)) - py_obj.c_ob_type = rffi.cast(PyObject, pto) + py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) ptr = rffi.cast(ADDR, py_obj) if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) @@ -116,7 +116,7 @@ try: w_obj = state.py_objects_r2w[ptr] except KeyError: - ref_type = ref.c_ob_type + ref_type = rffi.cast(PyObject, ref.c_ob_type) if ref != ref_type and space.is_w(from_ref(space, ref_type), space.w_str): return force_string(space, ref) else: @@ -142,7 +142,7 @@ state = space.fromcache(State) ptr = rffi.cast(ADDR, obj) if ptr not in state.py_objects_r2w and \ - space.is_w(from_ref(space, obj.c_ob_type), space.w_str): + space.is_w(from_ref(space, rffi.cast(PyObject, obj.c_ob_type)), space.w_str): # this is a half-allocated string, lets call the deallocator # without modifying the r2w/w2r dicts _Py_Dealloc(space, obj) @@ -186,7 +186,6 @@ from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.api import generic_cpy_call_dont_decref pto = obj.c_ob_type - pto = rffi.cast(PyTypeObjectPtr, pto) #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ # "'s type which is", rffi.charp2str(pto.c_tp_name) generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Mon Apr 5 02:47:15 2010 @@ -40,8 +40,10 @@ w_type = from_ref(space, pyo) w_func = space.getattr(w_type, space.wrap("__new__")) assert PyTuple_Check(space, w_args) - args_w = space.listview(w_args)[:] - args_w.insert(0, w_type) + args_w = space.int_w(space.len(w_args)) * [space.w_None] + for i in range(space.len(w_args)): + args_w[i+1] = space.getitem(w_args, space.wrap(i)) + args_w[0] = w_type w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Mon Apr 5 02:47:15 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, build_type_checkers, - PyObjectP, cpython_api_c) + PyObjectP, cpython_api_c, PyTypeObjectPtr) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef @@ -17,8 +17,8 @@ py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') py_str.c_buffer[buflen-1] = '\0' py_str.c_size = length - py_str.c_ob_type = make_ref(space, space.w_str) - return py_str + py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) + return py_str @cpython_api_c() def PyString_FromFormatV(): @@ -50,7 +50,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): - if from_ref(space, ref.c_ob_type) is space.w_str: + if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: ref = rffi.cast(PyStringObject, ref) return ref.c_size else: @@ -80,7 +80,7 @@ py_newstr = new_empty_str(space, newsize) except MemoryError: Py_DecRef(space, ref[0]) - ref[0] = lltype.nullptr(PyObject) + ref[0] = lltype.nullptr(PyObject.TO) raise to_cp = newsize oldsize = py_str.c_size Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Mon Apr 5 02:47:15 2010 @@ -292,7 +292,7 @@ @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): - pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) + pto = obj.c_ob_type base = pto this_func_ptr = llhelper(subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) @@ -310,7 +310,7 @@ @cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, obj): obj = rffi.cast(PyStringObject, obj) - pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) + pto = obj.c_ob_type if obj.c_buffer: lltype.free(obj.c_buffer, flavor="raw") obj_voidp = rffi.cast(rffi.VOIDP_real, obj) @@ -322,7 +322,7 @@ def type_dealloc(space, obj): state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) - type_pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) + type_pto = obj.c_ob_type base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython @@ -379,10 +379,10 @@ if bases_w: ref = make_ref(space, bases_w[0]) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) - pto.c_ob_type = make_ref(space, space.type(space.w_type)) + pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.type(space.w_type))) PyPyType_Ready(space, pto, w_type) else: - pto.c_ob_type = lltype.nullptr(PyObject.TO) + pto.c_ob_type = lltype.nullptr(PyTypeObjectPtr.TO) if space.is_w(w_type, space.w_object): pto.c_tp_basicsize = rffi.sizeof(PyObject.TO) elif space.is_w(w_type, space.w_type): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Mon Apr 5 02:47:15 2010 @@ -2,13 +2,12 @@ from pypy.rpython.lltypesystem.lltype import Ptr, FuncType, Void from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ + PyTypeObject, PyTypeObjectPtr from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef -PyTypeObject = lltype.ForwardReference() -PyTypeObjectPtr = lltype.Ptr(PyTypeObject) PyCFunction = Ptr(FuncType([PyObject, PyObject], PyObject)) P, FT, PyO = Ptr, FuncType, PyObject PyOPtr = Ptr(lltype.Array(PyO, hints={'nolength': True})) From fijal at codespeak.net Mon Apr 5 03:30:52 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 03:30:52 +0200 (CEST) Subject: [pypy-svn] r73398 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405013052.40610282B90@codespeak.net> Author: fijal Date: Mon Apr 5 03:30:50 2010 New Revision: 73398 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: Fix slot_tp_new Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Mon Apr 5 03:30:50 2010 @@ -40,14 +40,10 @@ w_type = from_ref(space, pyo) w_func = space.getattr(w_type, space.wrap("__new__")) assert PyTuple_Check(space, w_args) - args_w = space.int_w(space.len(w_args)) * [space.w_None] - for i in range(space.len(w_args)): - args_w[i+1] = space.getitem(w_args, space.wrap(i)) - args_w[0] = w_type + args_w = [w_type] + space.fixedview(w_args) w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) - PyWrapperFlag_KEYWORDS = 1 # adopted from typeobject.c From fijal at codespeak.net Mon Apr 5 03:32:56 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 03:32:56 +0200 (CEST) Subject: [pypy-svn] r73399 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100405013256.14B31282B90@codespeak.net> Author: fijal Date: Mon Apr 5 03:32:54 2010 New Revision: 73399 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c pypy/branch/cpython-extension/pypy/module/cpyext/misc.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.h pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: include more stuff from CPython. I believe I should test it while I'm at adding it... Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Mon Apr 5 03:32:54 2010 @@ -51,6 +51,7 @@ import pypy.module.cpyext.mapping import pypy.module.cpyext.iterator import pypy.module.cpyext.unicodeobject +import pypy.module.cpyext.misc # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 03:32:54 2010 @@ -556,7 +556,9 @@ include_dir / "pyerrors.c", include_dir / "modsupport.c", include_dir / "getargs.c", - include_dir / "stringobject.c"], + include_dir / "stringobject.c", + include_dir / "mysnprintf.c", + include_dir / "pythonrun.c"], separate_module_sources = [code], export_symbols=export_symbols_eci, **kwds Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Mon Apr 5 03:32:54 2010 @@ -12,6 +12,7 @@ #define SIZEOF_VOID_P sizeof(void *) #define WITH_DOC_STRINGS #define HAVE_UNICODE +#define INT_MAX (1 << (8 * sizeof(int) - 1)) /* Compat stuff */ #ifndef _WIN32 @@ -56,6 +57,7 @@ #include #include #include +#include #include "boolobject.h" #include "floatobject.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c Mon Apr 5 03:32:54 2010 @@ -0,0 +1,105 @@ +#include "Python.h" +#include + +/* snprintf() wrappers. If the platform has vsnprintf, we use it, else we + emulate it in a half-hearted way. Even if the platform has it, we wrap + it because platforms differ in what vsnprintf does in case the buffer + is too small: C99 behavior is to return the number of characters that + would have been written had the buffer not been too small, and to set + the last byte of the buffer to \0. At least MS _vsnprintf returns a + negative value instead, and fills the entire buffer with non-\0 data. + + The wrappers ensure that str[size-1] is always \0 upon return. + + PyOS_snprintf and PyOS_vsnprintf never write more than size bytes + (including the trailing '\0') into str. + + If the platform doesn't have vsnprintf, and the buffer size needed to + avoid truncation exceeds size by more than 512, Python aborts with a + Py_FatalError. + + Return value (rv): + + When 0 <= rv < size, the output conversion was unexceptional, and + rv characters were written to str (excluding a trailing \0 byte at + str[rv]). + + When rv >= size, output conversion was truncated, and a buffer of + size rv+1 would have been needed to avoid truncation. str[size-1] + is \0 in this case. + + When rv < 0, "something bad happened". str[size-1] is \0 in this + case too, but the rest of str is unreliable. It could be that + an error in format codes was detected by libc, or on platforms + with a non-C99 vsnprintf simply that the buffer wasn't big enough + to avoid truncation, or on platforms without any vsnprintf that + PyMem_Malloc couldn't obtain space for a temp buffer. + + CAUTION: Unlike C99, str != NULL and size > 0 are required. +*/ + +int +PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) +{ + int len; /* # bytes written, excluding \0 */ +#ifdef HAVE_SNPRINTF +#define _PyOS_vsnprintf_EXTRA_SPACE 1 +#else +#define _PyOS_vsnprintf_EXTRA_SPACE 512 + char *buffer; +#endif + assert(str != NULL); + assert(size > 0); + assert(format != NULL); + /* We take a size_t as input but return an int. Sanity check + * our input so that it won't cause an overflow in the + * vsnprintf return value or the buffer malloc size. */ + if (size > INT_MAX - _PyOS_vsnprintf_EXTRA_SPACE) { + len = -666; + goto Done; + } + +#ifdef HAVE_SNPRINTF + len = vsnprintf(str, size, format, va); +#else + /* Emulate it. */ + buffer = PyMem_MALLOC(size + _PyOS_vsnprintf_EXTRA_SPACE); + if (buffer == NULL) { + len = -666; + goto Done; + } + + len = vsprintf(buffer, format, va); + if (len < 0) + /* ignore the error */; + + else if ((size_t)len >= size + _PyOS_vsnprintf_EXTRA_SPACE) + Py_FatalError("Buffer overflow in PyOS_snprintf/PyOS_vsnprintf"); + + else { + const size_t to_copy = (size_t)len < size ? + (size_t)len : size - 1; + assert(to_copy < size); + memcpy(str, buffer, to_copy); + str[to_copy] = '\0'; + } + PyMem_FREE(buffer); +#endif +Done: + if (size > 0) + str[size-1] = '\0'; + return len; +#undef _PyOS_vsnprintf_EXTRA_SPACE +} + +int +PyOS_snprintf(char *str, size_t size, const char *format, ...) +{ + int rc; + va_list va; + + va_start(va, format); + rc = PyOS_vsnprintf(str, size, format, va); + va_end(va); + return rc; +} Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c Mon Apr 5 03:32:54 2010 @@ -0,0 +1,31 @@ + +#include "Python.h" + +void +Py_FatalError(const char *msg) +{ + fprintf(stderr, "Fatal Python error: %s\n", msg); + fflush(stderr); /* it helps in Windows debug build */ + +#ifdef MS_WINDOWS + { + size_t len = strlen(msg); + WCHAR* buffer; + size_t i; + + /* Convert the message to wchar_t. This uses a simple one-to-one + conversion, assuming that the this error message actually uses ASCII + only. If this ceases to be true, we will have to convert. */ + buffer = alloca( (len+1) * (sizeof *buffer)); + for( i=0; i<=len; ++i) + buffer[i] = msg[i]; + OutputDebugStringW(L"Fatal Python error: "); + OutputDebugStringW(buffer); + OutputDebugStringW(L"\n"); + } +#ifdef _DEBUG + DebugBreak(); +#endif +#endif /* MS_WINDOWS */ + abort(); +} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.h Mon Apr 5 03:32:54 2010 @@ -6,6 +6,7 @@ extern "C" { #endif + void Py_FatalError(const char *msg); #ifdef __cplusplus } Added: pypy/branch/cpython-extension/pypy/module/cpyext/misc.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/misc.py Mon Apr 5 03:32:54 2010 @@ -0,0 +1,14 @@ + +from pypy.module.cpyext.api import cpython_api_c + + at cpython_api_c() +def Py_FatalError(): + pass + + at cpython_api_c() +def PyOS_snprintf(): + pass + + at cpython_api_c() +def PyOS_vsnprintf(): + pass Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Mon Apr 5 03:32:54 2010 @@ -1,12 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cpython_api_c from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError + at cpython_api_c() +def PyModule_AddObject(): + pass + def PyImport_AddModule(space, name): w_name = space.wrap(name) w_mod = space.wrap(Module(space, w_name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Mon Apr 5 03:32:54 2010 @@ -1,7 +1,9 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, ll2ctypes + +from pypy.interpreter.gateway import interp2app class TestExceptions(BaseApiTest): def test_GivenExceptionMatches(self, space, api): @@ -55,6 +57,17 @@ class AppTestFetch(AppTestCpythonExtensionBase): + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + space = cls.space + + def set_errno(num): + import pdb + pdb.set_trace() + ll2ctypes.TLS.errno = num + + cls.w_set_errno = space.wrap(interp2app(set_errno, unwrap_spec=[int])) + def test_occurred(self): module = self.import_extension('foo', [ ("check_error", "METH_NOARGS", @@ -78,13 +91,12 @@ module = self.import_extension('foo', [ ("set_from_errno", "METH_NOARGS", ''' - int close(int); - close(-1); PyErr_SetFromErrno(PyExc_OSError); return NULL; '''), ]) try: + self.set_errno(errno.EBADF) module.set_from_errno() except OSError, e: assert e.errno == errno.EBADF From benjamin at codespeak.net Mon Apr 5 03:35:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Apr 2010 03:35:04 +0200 (CEST) Subject: [pypy-svn] r73400 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100405013504.A9E12282B90@codespeak.net> Author: benjamin Date: Mon Apr 5 03:35:03 2010 New Revision: 73400 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/modinit.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (props changed) Log: fix eol From fijal at codespeak.net Mon Apr 5 03:36:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 03:36:18 +0200 (CEST) Subject: [pypy-svn] r73401 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src Message-ID: <20100405013618.BEFFC282B90@codespeak.net> Author: fijal Date: Mon Apr 5 03:36:17 2010 New Revision: 73401 Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c - copied unchanged from r73397, pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c - copied unchanged from r73397, pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c pypy/branch/cpython-extension/pypy/module/cpyext/src/mysnprintf.c - copied unchanged from r73399, pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c pypy/branch/cpython-extension/pypy/module/cpyext/src/pyerrors.c - copied unchanged from r73397, pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c pypy/branch/cpython-extension/pypy/module/cpyext/src/pythonrun.c - copied unchanged from r73399, pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c - copied unchanged from r73397, pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c pypy/branch/cpython-extension/pypy/module/cpyext/src/varargwrapper.c - copied unchanged from r73397, pypy/branch/cpython-extension/pypy/module/cpyext/include/varargwrapper.c Removed: pypy/branch/cpython-extension/pypy/module/cpyext/include/getargs.c pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.c pypy/branch/cpython-extension/pypy/module/cpyext/include/mysnprintf.c pypy/branch/cpython-extension/pypy/module/cpyext/include/pyerrors.c pypy/branch/cpython-extension/pypy/module/cpyext/include/pythonrun.c pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.c pypy/branch/cpython-extension/pypy/module/cpyext/include/varargwrapper.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Actually move *.c files from include/ to src/ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 03:36:17 2010 @@ -34,6 +34,7 @@ ADDR = lltype.Signed include_dir = py.path.local(autopath.pypydir) / 'module' / 'cpyext' / 'include' +source_dir = py.path.local(autopath.pypydir) / 'module' / 'cpyext' / 'src' include_dirs = [ include_dir, udir, @@ -552,13 +553,13 @@ eci = ExternalCompilationInfo( include_dirs=include_dirs, - separate_module_files=[include_dir / "varargwrapper.c", - include_dir / "pyerrors.c", - include_dir / "modsupport.c", - include_dir / "getargs.c", - include_dir / "stringobject.c", - include_dir / "mysnprintf.c", - include_dir / "pythonrun.c"], + separate_module_files=[source_dir / "varargwrapper.c", + source_dir / "pyerrors.c", + source_dir / "modsupport.c", + source_dir / "getargs.c", + source_dir / "stringobject.c", + source_dir / "mysnprintf.c", + source_dir / "pythonrun.c"], separate_module_sources = [code], export_symbols=export_symbols_eci, **kwds From fijal at codespeak.net Mon Apr 5 03:37:19 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 03:37:19 +0200 (CEST) Subject: [pypy-svn] r73402 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100405013719.CC7F9282B90@codespeak.net> Author: fijal Date: Mon Apr 5 03:37:18 2010 New Revision: 73402 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: Remove pdb. This test is skipped anyway Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Mon Apr 5 03:37:18 2010 @@ -62,8 +62,6 @@ space = cls.space def set_errno(num): - import pdb - pdb.set_trace() ll2ctypes.TLS.errno = num cls.w_set_errno = space.wrap(interp2app(set_errno, unwrap_spec=[int])) From benjamin at codespeak.net Mon Apr 5 03:42:32 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Apr 2010 03:42:32 +0200 (CEST) Subject: [pypy-svn] r73403 - pypy/branch/cpython-extension/pypy/module/cpyext/src Message-ID: <20100405014232.8CC36282B90@codespeak.net> Author: benjamin Date: Mon Apr 5 03:42:31 2010 New Revision: 73403 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/mysnprintf.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/src/pythonrun.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c (props changed) Log: re set eol From benjamin at codespeak.net Mon Apr 5 03:49:04 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Apr 2010 03:49:04 +0200 (CEST) Subject: [pypy-svn] r73404 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405014904.51BB6282B90@codespeak.net> Author: benjamin Date: Mon Apr 5 03:49:02 2010 New Revision: 73404 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Log: simplify with call_method Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Mon Apr 5 03:49:02 2010 @@ -8,5 +8,4 @@ This is equivalent to the Python expression o.keys().""" # XXX: Cpython implements this in terms of PyObject_CallMethod, we should # do that eventually. - w_meth = space.getattr(w_obj, space.wrap("keys")) - return space.call_function(w_meth) + return space.call_method(w_obj, "keys") From fijall at gmail.com Mon Apr 5 04:24:16 2010 From: fijall at gmail.com (Maciej Fijalkowski) Date: Sun, 4 Apr 2010 20:24:16 -0600 Subject: [pypy-svn] r73394 - in pypy/branch/decouple-host-opcodes/pypy: . interpreter objspace/flow objspace/flow/test objspace/std tool In-Reply-To: <20100405001611.2752D282B90@codespeak.net> References: <20100405001611.2752D282B90@codespeak.net> Message-ID: Does this pass objspace/flow tests? I wonder if you can decouple them. On Sun, Apr 4, 2010 at 6:16 PM, wrote: > Author: antoine > Date: Mon Apr ?5 02:16:09 2010 > New Revision: 73394 > > Modified: > ? pypy/branch/decouple-host-opcodes/pypy/ ? (props changed) > ? pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py > ? pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py > ? pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py > ? pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py > ? pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py > ? pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py > ? pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py > ? pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py > ? pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py > Log: > Initial draft, using separate subclasses for pypy bytecode frames > and host bytecode frames. > Not sure it translates properly, although I've already fixed a > couple of issues. > > > > Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ?Mon Apr ?5 02:16:09 2010 > @@ -257,8 +257,13 @@ > ? ? ? ? self.actionflag.register_action(self.user_del_action) > ? ? ? ? self.actionflag.register_action(self.frame_trace_action) > > - ? ? ? ?from pypy.interpreter.pyframe import PyFrame > - ? ? ? ?self.FrameClass = PyFrame ? ?# can be overridden to a subclass > + ? ? ? ?from pypy.interpreter.pycode import cpython_magic, default_magic > + ? ? ? ?from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame > + ? ? ? ?self.our_magic = default_magic > + ? ? ? ?self.host_magic = cpython_magic > + ? ? ? ?# can be overridden to a subclass > + ? ? ? ?self.FrameClass = PyPyFrame > + ? ? ? ?self.HostFrameClass = HostPyFrame > > ? ? ? ? if self.config.objspace.logbytecodes: > ? ? ? ? ? ? self.bytecodecounts = [0] * 256 > @@ -589,7 +594,12 @@ > > ? ? def createframe(self, code, w_globals, closure=None): > ? ? ? ? "Create an empty PyFrame suitable for this code object." > - ? ? ? ?return self.FrameClass(self, code, w_globals, closure) > + ? ? ? ?magic = code.magic > + ? ? ? ?if magic == self.host_magic: > + ? ? ? ? ? ?return self.HostFrameClass(self, code, w_globals, closure) > + ? ? ? ?elif magic == self.our_magic: > + ? ? ? ? ? ?return self.FrameClass(self, code, w_globals, closure) > + ? ? ? ?raise ValueError("bad magic %s" % magic) > > ? ? def allocate_lock(self): > ? ? ? ? """Return an interp-level Lock object if threads are enabled, > > Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py ? ? ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/function.py ? ? ?Mon Apr ?5 02:16:09 2010 > @@ -236,9 +236,9 @@ > ? ? ? ? from pypy.interpreter.gateway import BuiltinCode > ? ? ? ? if isinstance(self.code, BuiltinCode): > ? ? ? ? ? ? identifier = self.code.identifier > - ? ? ? ? ? ?if Function._all.get(identifier, self) is not self: > - ? ? ? ? ? ? ? ?print "builtin code identifier %s used twice: %s and %s" % ( > - ? ? ? ? ? ? ? ? ? ?identifier, self, Function._all[identifier]) > + ? ? ? ? ? ?#if Function._all.get(identifier, self) is not self: > + ? ? ? ? ? ? ? ?#print "builtin code identifier %s used twice: %s and %s" % ( > + ? ? ? ? ? ? ? ? ? ?#identifier, self, Function._all[identifier]) > ? ? ? ? ? ? # we have been seen by other means so rtyping should not choke > ? ? ? ? ? ? # on us > ? ? ? ? ? ? Function._all[identifier] = self > > Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py ? ? ? (original) > +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py ? ? ? Mon Apr ?5 02:16:09 2010 > @@ -135,8 +135,9 @@ > ? ? ? ? from pypy.rlib import rstack > ? ? ? ? # the following 'assert' is an annotation hint: it hides from > ? ? ? ? # the annotator all methods that are defined in PyFrame but > - ? ? ? ?# overridden in the FrameClass subclass of PyFrame. > - ? ? ? ?assert isinstance(self, self.space.FrameClass) > + ? ? ? ?# overridden in the {,Host}FrameClass subclasses of PyFrame. > + ? ? ? ?assert (isinstance(self, self.space.FrameClass) > + ? ? ? ? ? ? or isinstance(self, self.space.HostFrameClass)) > ? ? ? ? executioncontext = self.space.getexecutioncontext() > ? ? ? ? executioncontext.enter(self) > ? ? ? ? try: > @@ -628,6 +629,13 @@ > ? ? ? ? ? ? return space.wrap(self.builtin is not space.builtin) > ? ? ? ? return space.w_False > > + > +class PyPyFrame(PyFrame): > + ? ?pass > + > +class HostPyFrame(PyFrame): > + ? ?pass > + > ?# ____________________________________________________________ > > ?def get_block_class(opname): > > Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ? ? ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ? ? ?Mon Apr ?5 02:16:09 2010 > @@ -14,9 +14,9 @@ > ?from pypy.rlib import jit, rstackovf > ?from pypy.rlib.rarithmetic import r_uint, intmask > ?from pypy.rlib.unroll import unrolling_iterable > -from pypy.tool.stdlib_opcode import (opcodedesc, HAVE_ARGUMENT, > - ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unrolling_opcode_descs, > - ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? opcode_method_names) > +from pypy.tool.stdlib_opcode import (unrolling_opcode_descs, > + ? ?HAVE_ARGUMENT, host_HAVE_ARGUMENT, opcodedesc, host_opcodedesc, > + ? ?opcode_method_names, host_opcode_method_names, ) > > ?def unaryoperation(operationname): > ? ? """NOT_RPYTHON""" > @@ -180,7 +180,7 @@ > ? ? ? ? ? ? ? ? probs[opcode] = probs.get(opcode, 0) + 1 > ? ? ? ? ? ? ? ? self.last_opcode = opcode > > - ? ? ? ? ? ?if opcode >= HAVE_ARGUMENT: > + ? ? ? ? ? ?if opcode >= self.HAVE_ARGUMENT: > ? ? ? ? ? ? ? ? lo = ord(co_code[next_instr]) > ? ? ? ? ? ? ? ? hi = ord(co_code[next_instr+1]) > ? ? ? ? ? ? ? ? next_instr += 2 > @@ -188,16 +188,16 @@ > ? ? ? ? ? ? else: > ? ? ? ? ? ? ? ? oparg = 0 > > - ? ? ? ? ? ?while opcode == opcodedesc.EXTENDED_ARG.index: > + ? ? ? ? ? ?while opcode == self.opcodedesc.EXTENDED_ARG.index: > ? ? ? ? ? ? ? ? opcode = ord(co_code[next_instr]) > - ? ? ? ? ? ? ? ?if opcode < HAVE_ARGUMENT: > + ? ? ? ? ? ? ? ?if opcode < self.HAVE_ARGUMENT: > ? ? ? ? ? ? ? ? ? ? raise BytecodeCorruption > ? ? ? ? ? ? ? ? lo = ord(co_code[next_instr+1]) > ? ? ? ? ? ? ? ? hi = ord(co_code[next_instr+2]) > ? ? ? ? ? ? ? ? next_instr += 3 > ? ? ? ? ? ? ? ? oparg = (oparg << 16) | (hi << 8) | lo > > - ? ? ? ? ? ?if opcode == opcodedesc.RETURN_VALUE.index: > + ? ? ? ? ? ?if opcode == self.opcodedesc.RETURN_VALUE.index: > ? ? ? ? ? ? ? ? w_returnvalue = self.popvalue() > ? ? ? ? ? ? ? ? block = self.unrollstack(SReturnValue.kind) > ? ? ? ? ? ? ? ? if block is None: > @@ -208,11 +208,11 @@ > ? ? ? ? ? ? ? ? ? ? next_instr = block.handle(self, unroller) > ? ? ? ? ? ? ? ? ? ? return next_instr ? ?# now inside a 'finally' block > > - ? ? ? ? ? ?if opcode == opcodedesc.YIELD_VALUE.index: > + ? ? ? ? ? ?if opcode == self.opcodedesc.YIELD_VALUE.index: > ? ? ? ? ? ? ? ? #self.last_instr = intmask(next_instr - 1) XXX clean up! > ? ? ? ? ? ? ? ? raise Yield > > - ? ? ? ? ? ?if opcode == opcodedesc.END_FINALLY.index: > + ? ? ? ? ? ?if opcode == self.opcodedesc.END_FINALLY.index: > ? ? ? ? ? ? ? ? unroller = self.end_finally() > ? ? ? ? ? ? ? ? if isinstance(unroller, SuspendedUnroller): > ? ? ? ? ? ? ? ? ? ? # go on unrolling the stack > @@ -225,7 +225,7 @@ > ? ? ? ? ? ? ? ? ? ? ? ? next_instr = block.handle(self, unroller) > ? ? ? ? ? ? ? ? return next_instr > > - ? ? ? ? ? ?if opcode == opcodedesc.JUMP_ABSOLUTE.index: > + ? ? ? ? ? ?if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: > ? ? ? ? ? ? ? ? return self.jump_absolute(oparg, next_instr, ec) > > ? ? ? ? ? ? if we_are_translated(): > @@ -242,7 +242,7 @@ > ? ? ? ? ? ? ? ? ? ? ? ? # dispatch to the opcode method > ? ? ? ? ? ? ? ? ? ? ? ? meth = getattr(self, opdesc.methodname) > ? ? ? ? ? ? ? ? ? ? ? ? res = meth(oparg, next_instr) > - ? ? ? ? ? ? ? ? ? ? ? ?if opdesc.index == opcodedesc.CALL_FUNCTION.index: > + ? ? ? ? ? ? ? ? ? ? ? ?if opdesc.index == self.opcodedesc.CALL_FUNCTION.index: > ? ? ? ? ? ? ? ? ? ? ? ? ? ? rstack.resume_point("dispatch_call", self, co_code, next_instr, ec) > ? ? ? ? ? ? ? ? ? ? ? ? # !! warning, for the annotator the next line is not > ? ? ? ? ? ? ? ? ? ? ? ? # comparing an int and None - you can't do that. > @@ -254,7 +254,7 @@ > ? ? ? ? ? ? ? ? ? ? self.MISSING_OPCODE(oparg, next_instr) > > ? ? ? ? ? ? else: ?# when we are not translated, a list lookup is much faster > - ? ? ? ? ? ? ? ?methodname = opcode_method_names[opcode] > + ? ? ? ? ? ? ? ?methodname = self.opcode_method_names[opcode] > ? ? ? ? ? ? ? ? res = getattr(self, methodname)(oparg, next_instr) > ? ? ? ? ? ? ? ? if res is not None: > ? ? ? ? ? ? ? ? ? ? next_instr = res > @@ -690,26 +690,6 @@ > ? ? ? ? w_list = self.space.newlist(items) > ? ? ? ? self.pushvalue(w_list) > > - ? ?def BUILD_MAP(self, itemcount, next_instr): > - ? ? ? ?if not we_are_translated() and sys.version_info >= (2, 6): > - ? ? ? ? ? ?# We could pre-allocate a dict here > - ? ? ? ? ? ?# but for the moment this code is not translated. > - ? ? ? ? ? ?pass > - ? ? ? ?else: > - ? ? ? ? ? ?if itemcount != 0: > - ? ? ? ? ? ? ? ?raise BytecodeCorruption > - ? ? ? ?w_dict = self.space.newdict() > - ? ? ? ?self.pushvalue(w_dict) > - > - ? ?def STORE_MAP(self, zero, next_instr): > - ? ? ? ?if not we_are_translated() and sys.version_info >= (2, 6): > - ? ? ? ? ? ?w_key = self.popvalue() > - ? ? ? ? ? ?w_value = self.popvalue() > - ? ? ? ? ? ?w_dict = self.peekvalue() > - ? ? ? ? ? ?self.space.setitem(w_dict, w_key, w_value) > - ? ? ? ?else: > - ? ? ? ? ? ?raise BytecodeCorruption > - > ? ? def LOAD_ATTR(self, nameindex, next_instr): > ? ? ? ? "obj.attributename" > ? ? ? ? w_attributename = self.getname_w(nameindex) > @@ -1020,6 +1000,48 @@ > ? ? STOP_CODE = MISSING_OPCODE > > > +class __extend__(pyframe.PyPyFrame): > + ? ?opcode_method_names = opcode_method_names > + ? ?opcodedesc = opcodedesc > + ? ?HAVE_ARGUMENT = HAVE_ARGUMENT > + > + ? ?def BUILD_MAP(self, itemcount, next_instr): > + ? ? ? ?if itemcount != 0: > + ? ? ? ? ? ?raise BytecodeCorruption > + ? ? ? ?w_dict = self.space.newdict() > + ? ? ? ?self.pushvalue(w_dict) > + > + ? ?def STORE_MAP(self, zero, next_instr): > + ? ? ? ?raise BytecodeCorruption > + > +host_version_info = sys.version_info > + > +class __extend__(pyframe.HostPyFrame): > + ? ?opcode_method_names = host_opcode_method_names > + ? ?opcodedesc = host_opcodedesc > + ? ?HAVE_ARGUMENT = host_HAVE_ARGUMENT > + > + ? ?def BUILD_MAP(self, itemcount, next_instr): > + ? ? ? ?if host_version_info >= (2, 6): > + ? ? ? ? ? ?# We could pre-allocate a dict here > + ? ? ? ? ? ?# but for the moment this code is not translated. > + ? ? ? ? ? ?pass > + ? ? ? ?else: > + ? ? ? ? ? ?if itemcount != 0: > + ? ? ? ? ? ? ? ?raise BytecodeCorruption > + ? ? ? ?w_dict = self.space.newdict() > + ? ? ? ?self.pushvalue(w_dict) > + > + ? ?def STORE_MAP(self, zero, next_instr): > + ? ? ? ?if host_version_info >= (2, 6): > + ? ? ? ? ? ?w_key = self.popvalue() > + ? ? ? ? ? ?w_value = self.popvalue() > + ? ? ? ? ? ?w_dict = self.peekvalue() > + ? ? ? ? ? ?self.space.setitem(w_dict, w_key, w_value) > + ? ? ? ?else: > + ? ? ? ? ? ?raise BytecodeCorruption > + > + > ?### ____________________________________________________________ ### > > ?class ExitFrame(Exception): > > Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py (original) > +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py Mon Apr ?5 02:16:09 2010 > @@ -1,7 +1,7 @@ > ?import collections > ?from pypy.interpreter.executioncontext import ExecutionContext > ?from pypy.interpreter.error import OperationError > -from pypy.interpreter import pyframe > +from pypy.interpreter import pyframe, pycode > ?from pypy.interpreter.argument import ArgumentsForTranslation > ?from pypy.objspace.flow.model import * > ?from pypy.objspace.flow.framestate import FrameState > @@ -219,8 +219,12 @@ > ? ? ? ? # create an empty frame suitable for the code object > ? ? ? ? # while ignoring any operation like the creation of the locals dict > ? ? ? ? self.recorder = [] > - ? ? ? ?frame = FlowSpaceFrame(self.space, self.code, > - ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? self.w_globals, self.closure) > + ? ? ? ?if self.code.magic == pycode.cpython_magic: > + ? ? ? ? ? ?frame = FlowSpaceHostPyFrame(self.space, self.code, > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? self.w_globals, self.closure) > + ? ? ? ?else: > + ? ? ? ? ? ?frame = FlowSpacePyPyFrame(self.space, self.code, > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? self.w_globals, self.closure) > ? ? ? ? frame.last_instr = 0 > ? ? ? ? return frame > > @@ -398,7 +402,7 @@ > ? ? ? ? ? ? ? ? if w_v.value is oldvalue: > ? ? ? ? ? ? ? ? ? ? stack_items_w[i] = w_new > > -class FlowSpaceFrame(pyframe.PyFrame): > +class FlowSpaceFrameBase(object): > ? ? def make_arguments(self, nargs): > ? ? ? ? return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) > ? ? def argument_factory(self, *args): > @@ -410,3 +414,10 @@ > ? ? ? ? ? ? raise operr > ? ? ? ? return pyframe.PyFrame.handle_operation_error(self, ec, operr, > ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? *args, **kwds) > + > +class FlowSpacePyPyFrame(FlowSpaceFrameBase, pyframe.PyPyFrame): > + ? ?pass > + > +class FlowSpaceHostPyFrame(FlowSpaceFrameBase, pyframe.HostPyFrame): > + ? ?pass > + > > Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py ?Mon Apr ?5 02:16:09 2010 > @@ -826,23 +826,34 @@ > ? ? ? ? """ Tests code generated by pypy-c compiled with CALL_METHOD > ? ? ? ? bytecode > ? ? ? ? """ > - ? ? ? ?class X: > - ? ? ? ? ? ?def m(self): > - ? ? ? ? ? ? ? ?return 3 > - > - ? ? ? ?def f(): > - ? ? ? ? ? ?x = X() > - ? ? ? ? ? ?return x.m() > - > - ? ? ? ?# this code is generated by pypy-c when compiling above f > - ? ? ? ?pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' > - ? ? ? ?new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) > - ? ? ? ?f2 = new.function(new_c, locals(), 'f') > - > - ? ? ? ?graph = self.codetest(f2) > - ? ? ? ?all_ops = self.all_operations(graph) > - ? ? ? ?assert all_ops['simple_call'] == 2 > - ? ? ? ?assert all_ops['getattr'] == 1 > + ? ? ? ?from pypy.objspace.flow import flowcontext > + ? ? ? ?old_class = flowcontext.FlowSpaceHostPyFrame > + ? ? ? ?try: > + ? ? ? ? ? ?# HACK: we will replace a host-generated code object with a > + ? ? ? ? ? ?# pypy-generated code object, but it will carry the host's > + ? ? ? ? ? ?# magic number (since it's generated with the host's code.new). > + ? ? ? ? ? ?flowcontext.FlowSpaceHostPyFrame = flowcontext.FlowSpacePyPyFrame > + ? ? ? ? ? ?class X: > + ? ? ? ? ? ? ? ?def m(self): > + ? ? ? ? ? ? ? ? ? ?return 3 > + > + ? ? ? ? ? ?def f(): > + ? ? ? ? ? ? ? ?x = X() > + ? ? ? ? ? ? ? ?return x.m() > + > + ? ? ? ? ? ?# this code is generated by pypy-c when compiling above f > + ? ? ? ? ? ?pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' > + ? ? ? ? ? ?new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) > + ? ? ? ? ? ?f2 = new.function(new_c, locals(), 'f') > + ? ? ? ? ? ?import dis > + ? ? ? ? ? ?dis.dis(f2) > + > + ? ? ? ? ? ?graph = self.codetest(f2) > + ? ? ? ? ? ?all_ops = self.all_operations(graph) > + ? ? ? ? ? ?assert all_ops['simple_call'] == 2 > + ? ? ? ? ? ?assert all_ops['getattr'] == 1 > + ? ? ? ?finally: > + ? ? ? ? ? ?flowcontext.FlowSpaceHostPyFrame = old_class > > ? ? def test_generator(self): > ? ? ? ? def f(): > > Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py ? ? ? ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/frame.py ? ? ? ?Mon Apr ?5 02:16:09 2010 > @@ -3,14 +3,16 @@ > ?import operator > > ?from pypy.rlib.unroll import unrolling_iterable > -from pypy.interpreter import pyframe, pyopcode, function > +from pypy.interpreter import pyopcode, function > ?from pypy.interpreter.error import OperationError, operationerrfmt > ?from pypy.module.__builtin__ import OPTIMIZED_BUILTINS, Module > ?from pypy.objspace.std.multimethod import FailedToImplement > > > -class BaseFrame(pyframe.PyFrame): > + > +class BaseFrame(object): > ? ? """These opcodes are always overridden.""" > + ? ?_mixin_ = True > > ? ? def LIST_APPEND(f, oparg, next_instr): > ? ? ? ? from pypy.objspace.std.listobject import W_ListObject > @@ -135,9 +137,9 @@ > ? ? f.pushvalue(w_result) > > > -def build_frame(space): > +def build_frame(space, baseclass): > ? ? """Consider the objspace config and return a patched frame object.""" > - ? ?class StdObjSpaceFrame(BaseFrame): > + ? ?class StdObjSpaceFrame(baseclass, BaseFrame): > ? ? ? ? pass > ? ? if space.config.objspace.std.optimized_int_add: > ? ? ? ? if space.config.objspace.std.withsmallint: > > Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py ? ? (original) > +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py ? ? Mon Apr ?5 02:16:09 2010 > @@ -47,7 +47,9 @@ > ? ? ? ? # setup all the object types and implementations > ? ? ? ? self.model = model.StdTypeModel(self.config) > > - ? ? ? ?self.FrameClass = frame.build_frame(self) > + ? ? ? ?from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame > + ? ? ? ?self.FrameClass = frame.build_frame(self, PyPyFrame) > + ? ? ? ?self.HostFrameClass = frame.build_frame(self, HostPyFrame) > > ? ? ? ? if self.config.objspace.std.withrope: > ? ? ? ? ? ? self.StringObjectCls = W_RopeObject > @@ -133,7 +135,7 @@ > ? ? ? ? if not we_are_translated() and isinstance(code, CPythonFakeCode): > ? ? ? ? ? ? return CPythonFakeFrame(self, code, w_globals) > ? ? ? ? else: > - ? ? ? ? ? ?return self.FrameClass(self, code, w_globals, closure) > + ? ? ? ? ? ?return ObjSpace.createframe(self, code, w_globals, closure) > > ? ? def gettypefor(self, cls): > ? ? ? ? return self.gettypeobject(cls.typedef) > > Modified: pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py > ============================================================================== > --- pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py ? ? ? ?(original) > +++ pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py ? ? ? ?Mon Apr ?5 02:16:09 2010 > @@ -4,6 +4,9 @@ > ? ? ? ? ? ?'hasconst', 'hasname', 'hasjrel', 'hasjabs', > ? ? ? ? ? ?'haslocal', 'hascompare', 'hasfree', 'cmp_op'] > > +from opcode import ( > + ? ?opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) > + > ?def load_opcode(): > ? ? import py > ? ? opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') > @@ -23,65 +26,83 @@ > ?STORE_SLICE = opmap["STORE_SLICE+0"] > ?DELETE_SLICE = opmap["DELETE_SLICE+0"] > > -opcode_method_names = ['MISSING_OPCODE'] * 256 > -for name, index in opmap.items(): > - ? ?opcode_method_names[index] = name.replace('+', '_') > +def make_method_names(opmap): > + ? ?tbl = ['MISSING_OPCODE'] * 256 > + ? ?for name, index in opmap.items(): > + ? ? ? ?tbl[index] = name.replace('+', '_') > + ? ?return tbl > + > +opcode_method_names = make_method_names(opmap) > +host_opcode_method_names = make_method_names(host_opmap) > +#print ( > + ? ?#set(enumerate(opcode_method_names)) ^ set(enumerate(host_opcode_method_names)) > +#) > +del make_method_names > > ?# ____________________________________________________________ > ?# RPython-friendly helpers and structures > > ?from pypy.rlib.unroll import unrolling_iterable > > +def make_opcode_desc(HAVE_ARGUMENT): > + ? ?class OpcodeDesc(object): > + ? ? ? ?def __init__(self, name, index): > + ? ? ? ? ? ?self.name = name > + ? ? ? ? ? ?self.methodname = opcode_method_names[index] > + ? ? ? ? ? ?self.index = index > + ? ? ? ? ? ?self.hasarg = index >= HAVE_ARGUMENT > + > + ? ? ? ?def _freeze_(self): > + ? ? ? ? ? ?return True > + > + ? ? ? ?def is_enabled(self, space): > + ? ? ? ? ? ?"""Check if the opcode should be enabled in the space's configuration. > + ? ? ? ? ? ?(Returns True for all standard opcodes.)""" > + ? ? ? ? ? ?opt = space.config.objspace.opcodes > + ? ? ? ? ? ?return getattr(opt, self.name, True) > + ? ? ? ?is_enabled._annspecialcase_ = 'specialize:memo' > + > + ? ? ? ?# for predictable results, we try to order opcodes most-used-first > + ? ? ? ?opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] > + > + ? ? ? ?def sortkey(self): > + ? ? ? ? ? ?try: > + ? ? ? ? ? ? ? ?i = self.opcodeorder.index(self.index) > + ? ? ? ? ? ?except ValueError: > + ? ? ? ? ? ? ? ?i = 1000000 > + ? ? ? ? ? ?return i, self.index > + > + ? ? ? ?def __cmp__(self, other): > + ? ? ? ? ? ?return (cmp(self.__class__, other.__class__) or > + ? ? ? ? ? ? ? ? ? ?cmp(self.sortkey(), other.sortkey())) > > -class OpcodeDesc(object): > - ? ?def __init__(self, name, index): > - ? ? ? ?self.name = name > - ? ? ? ?self.methodname = opcode_method_names[index] > - ? ? ? ?self.index = index > - ? ? ? ?self.hasarg = index >= HAVE_ARGUMENT > - > - ? ?def _freeze_(self): > - ? ? ? ?return True > - > - ? ?def is_enabled(self, space): > - ? ? ? ?"""Check if the opcode should be enabled in the space's configuration. > - ? ? ? ?(Returns True for all standard opcodes.)""" > - ? ? ? ?opt = space.config.objspace.opcodes > - ? ? ? ?return getattr(opt, self.name, True) > - ? ?is_enabled._annspecialcase_ = 'specialize:memo' > - > - ? ?# for predictable results, we try to order opcodes most-used-first > - ? ?opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] > - > - ? ?def sortkey(self): > - ? ? ? ?try: > - ? ? ? ? ? ?i = self.opcodeorder.index(self.index) > - ? ? ? ?except ValueError: > - ? ? ? ? ? ?i = 1000000 > - ? ? ? ?return i, self.index > - > - ? ?def __cmp__(self, other): > - ? ? ? ?return (cmp(self.__class__, other.__class__) or > - ? ? ? ? ? ? ? ?cmp(self.sortkey(), other.sortkey())) > + ? ?return OpcodeDesc > + > +OpcodeDesc = make_opcode_desc(HAVE_ARGUMENT) > +HostOpcodeDesc = make_opcode_desc(host_HAVE_ARGUMENT) > > ?opdescmap = {} > > -class opcodedesc: > +class _baseopcodedesc: > + ? ?pass > + > +class opcodedesc(_baseopcodedesc): > ? ? """A namespace mapping OPCODE_NAME to OpcodeDescs.""" > > +class host_opcodedesc(_baseopcodedesc): > + ? ?"""A namespace mapping OPCODE_NAME to HostOpcodeDescs.""" > + > ?for name, index in opmap.items(): > ? ? desc = OpcodeDesc(name, index) > ? ? setattr(opcodedesc, name, desc) > ? ? opdescmap[index] = desc > > +for name, index in host_opmap.items(): > + ? ?desc = HostOpcodeDesc(name, index) > + ? ?setattr(host_opcodedesc, name, desc) > + > ?lst = opdescmap.values() > ?lst.sort() > ?unrolling_opcode_descs = unrolling_iterable(lst) > > -# Allow non-translated code to interpret the new 2.6 bytecodes > -import sys > -if sys.version_info >= (2, 6): > - ? ?import opcode > - ? ?opcode_method_names[opcode.opmap['STORE_MAP']] = 'STORE_MAP' > - > ?del name, index, desc, lst > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From benjamin at codespeak.net Mon Apr 5 04:50:49 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 5 Apr 2010 04:50:49 +0200 (CEST) Subject: [pypy-svn] r73405 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100405025049.642CA282B90@codespeak.net> Author: benjamin Date: Mon Apr 5 04:50:46 2010 New Revision: 73405 Modified: pypy/trunk/pypy/annotation/bookkeeper.py pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py Log: revert r73347, which is causing bz2 failures for some reason Modified: pypy/trunk/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/trunk/pypy/annotation/bookkeeper.py (original) +++ pypy/trunk/pypy/annotation/bookkeeper.py Mon Apr 5 04:50:46 2010 @@ -523,7 +523,7 @@ self.getdesc(pyobj.im_func), # funcdesc self.getuniqueclassdef(origincls), # originclassdef classdef, # selfclassdef - name, pyobj=pyobj) + name) else: # must be a frozen pre-built constant, but let's check try: @@ -552,7 +552,7 @@ return result def getmethoddesc(self, funcdesc, originclassdef, selfclassdef, name, - flags={}, pyobj=None): + flags={}): flagskey = flags.items() flagskey.sort() key = funcdesc, originclassdef, selfclassdef, name, tuple(flagskey) @@ -560,7 +560,7 @@ return self.methoddescs[key] except KeyError: result = description.MethodDesc(self, funcdesc, originclassdef, - selfclassdef, name, flags, pyobj) + selfclassdef, name, flags) self.methoddescs[key] = result return result Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Mon Apr 5 04:50:46 2010 @@ -689,8 +689,8 @@ knowntype = types.MethodType def __init__(self, bookkeeper, funcdesc, originclassdef, - selfclassdef, name, flags={}, pyobj=None): - super(MethodDesc, self).__init__(bookkeeper, pyobj) + selfclassdef, name, flags={}): + super(MethodDesc, self).__init__(bookkeeper) self.funcdesc = funcdesc self.originclassdef = originclassdef self.selfclassdef = selfclassdef Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Mon Apr 5 04:50:46 2010 @@ -1751,23 +1751,6 @@ s = a.build_types(f, [bool]) assert s == annmodel.SomeString(can_be_None=True) - def test_method_hasattr(self): - class X: - def m(self): - return 4 - m.attr = 23 - def x(self, string): - return string - x = X() - m = x.m - def f(string): - if hasattr(m, "attr"): - return x.x(string) - return x.m() - a = self.RPythonAnnotator() - s = a.build_types(f, [str]) - assert s == annmodel.SomeString() - def test_dont_see_AttributeError_clause(self): class Stuff: def _freeze_(self): From fijal at codespeak.net Mon Apr 5 05:14:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 05:14:54 +0200 (CEST) Subject: [pypy-svn] r73406 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100405031454.79D74282B90@codespeak.net> Author: fijal Date: Mon Apr 5 05:14:52 2010 New Revision: 73406 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Support ':'. Copying code from C seems like a lot of hassle, since we need to support PyCObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Mon Apr 5 05:14:52 2010 @@ -20,8 +20,10 @@ c = fmt[i] if c == "\x00": return 1 - if c == "i": + elif c == "i": arr = api.va_get_int_star(va_list_p) arr[0] = rffi.cast(rffi.INT, space.int_w(space.getitem(w_obj, space.wrap(i)))) + elif c == ':': + return 1 i += 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Mon Apr 5 05:14:52 2010 @@ -13,7 +13,15 @@ } return PyInt_FromLong(l); ''' - )]) + ), + ('oneargandform', 'METH_VARARGS', + ''' + int l; + if (!PyArg_ParseTuple(args, "i:oneargandstuff", &l)) { + return NULL; + } + return PyInt_FromLong(l); + ''')]) assert mod.oneargint(1) == 1 raises(TypeError, mod.oneargint, None) try: @@ -22,3 +30,5 @@ pass else: raise Exception("DID NOT RAISE") + assert mod.oneargandform(1) == 1 + From fijal at codespeak.net Mon Apr 5 05:34:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 05:34:16 +0200 (CEST) Subject: [pypy-svn] r73407 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src Message-ID: <20100405033416.6CA60282B90@codespeak.net> Author: fijal Date: Mon Apr 5 05:34:14 2010 New Revision: 73407 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c Log: Provide a bit more support for Py_BuildValue and a couple of simple changes. WARNING: this is still a bit preliminary since we disable certain functions out of Py_BuildValue (long long and complex to be precise) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Mon Apr 5 05:34:14 2010 @@ -12,6 +12,10 @@ def PyArg_ParseTuple(): pass + at cpython_api_c() +def PyArg_UnpackTuple(): + pass + @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=0) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Mon Apr 5 05:34:14 2010 @@ -13,6 +13,7 @@ #define WITH_DOC_STRINGS #define HAVE_UNICODE #define INT_MAX (1 << (8 * sizeof(int) - 1)) +#define WITHOUT_COMPLEX /* Compat stuff */ #ifndef _WIN32 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/listobject.h Mon Apr 5 05:34:14 2010 @@ -1 +1,2 @@ #define PyList_GET_ITEM PyList_GetItem +#define PyList_SET_ITEM PyList_SetItem Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Mon Apr 5 05:34:14 2010 @@ -29,6 +29,12 @@ int PyModule_AddObject(PyObject *m, const char *name, PyObject *o); +PyObject * Py_BuildValue(const char *, ...); +PyObject * _Py_BuildValue_SizeT(const char *, ...); +int _PyArg_NoKeywords(const char *funcname, PyObject *kw); + +int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...); + /* * This is from pyport.h. Perhaps it belongs elsewhere. */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Mon Apr 5 05:34:14 2010 @@ -11,6 +11,10 @@ def PyModule_AddObject(): pass + at cpython_api_c() +def Py_BuildValue(): + pass + def PyImport_AddModule(space, name): w_name = space.wrap(name) w_mod = space.wrap(Module(space, w_name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Mon Apr 5 05:34:14 2010 @@ -137,5 +137,63 @@ return pypy_vgetargs1(args, format, &lva, FLAG_SIZE_T); } +int +PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) +{ + Py_ssize_t i, l; + PyObject **o; + va_list vargs; +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, max); +#else + va_start(vargs); +#endif + + assert(min >= 0); + assert(min <= max); + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_SystemError, + "PyArg_UnpackTuple() argument list is not a tuple"); + return 0; + } + l = PyTuple_GET_SIZE(args); + if (l < min) { + if (name != NULL) + PyErr_Format( + PyExc_TypeError, + "%s expected %s%zd arguments, got %zd", + name, (min == max ? "" : "at least "), min, l); + else + PyErr_Format( + PyExc_TypeError, + "unpacked tuple should have %s%zd elements," + " but has %zd", + (min == max ? "" : "at least "), min, l); + va_end(vargs); + return 0; + } + if (l > max) { + if (name != NULL) + PyErr_Format( + PyExc_TypeError, + "%s expected %s%zd arguments, got %zd", + name, (min == max ? "" : "at most "), max, l); + else + PyErr_Format( + PyExc_TypeError, + "unpacked tuple should have %s%zd elements," + " but has %zd", + (min == max ? "" : "at most "), max, l); + va_end(vargs); + return 0; + } + for (i = 0; i < l; i++) { + o = va_arg(vargs, PyObject **); + *o = PyTuple_GET_ITEM(args, i); + } + va_end(vargs); + return 1; + +} // REST IS NOT COPIED FROM CPYTHON Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c Mon Apr 5 05:34:14 2010 @@ -1,4 +1,522 @@ -#include + +/* Module support implementation */ + +#include "Python.h" + +#define FLAG_SIZE_T 1 +typedef double va_double; + +static PyObject *va_build_value(const char *, va_list, int); + +/* Package context -- the full module name for package imports */ +char *_Py_PackageContext = NULL; + +/* Py_InitModule4() parameters: + - name is the module name + - methods is the list of top-level functions + - doc is the documentation string + - passthrough is passed as self to functions defined in the module + - api_version is the value of PYTHON_API_VERSION at the time the + module was compiled + + Return value is a borrowed reference to the module object; or NULL + if an error occurred (in Python 1.4 and before, errors were fatal). + Errors may still leak memory. +*/ + +static char api_version_warning[] = +"Python C API version mismatch for module %.100s:\ + This Python has API version %d, module %.100s has version %d."; + +/* Helper for mkvalue() to scan the length of a format */ + +static int +countformat(const char *format, int endchar) +{ + int count = 0; + int level = 0; + while (level > 0 || *format != endchar) { + switch (*format) { + case '\0': + /* Premature end */ + PyErr_SetString(PyExc_SystemError, + "unmatched paren in format"); + return -1; + case '(': + case '[': + case '{': + if (level == 0) + count++; + level++; + break; + case ')': + case ']': + case '}': + level--; + break; + case '#': + case '&': + case ',': + case ':': + case ' ': + case '\t': + break; + default: + if (level == 0) + count++; + } + format++; + } + return count; +} + + +/* Generic function to create a value -- the inverse of getargs() */ +/* After an original idea and first implementation by Steven Miale */ + +static PyObject *do_mktuple(const char**, va_list *, int, int, int); +static PyObject *do_mklist(const char**, va_list *, int, int, int); +static PyObject *do_mkdict(const char**, va_list *, int, int, int); +static PyObject *do_mkvalue(const char**, va_list *, int); + + +static PyObject * +do_mkdict(const char **p_format, va_list *p_va, int endchar, int n, int flags) +{ + PyObject *d; + int i; + int itemfailed = 0; + if (n < 0) + return NULL; + if ((d = PyDict_New()) == NULL) + return NULL; + /* Note that we can't bail immediately on error as this will leak + refcounts on any 'N' arguments. */ + for (i = 0; i < n; i+= 2) { + PyObject *k, *v; + int err; + k = do_mkvalue(p_format, p_va, flags); + if (k == NULL) { + itemfailed = 1; + Py_INCREF(Py_None); + k = Py_None; + } + v = do_mkvalue(p_format, p_va, flags); + if (v == NULL) { + itemfailed = 1; + Py_INCREF(Py_None); + v = Py_None; + } + err = PyDict_SetItem(d, k, v); + Py_DECREF(k); + Py_DECREF(v); + if (err < 0 || itemfailed) { + Py_DECREF(d); + return NULL; + } + } + if (d != NULL && **p_format != endchar) { + Py_DECREF(d); + d = NULL; + PyErr_SetString(PyExc_SystemError, + "Unmatched paren in format"); + } + else if (endchar) + ++*p_format; + return d; +} + +static PyObject * +do_mklist(const char **p_format, va_list *p_va, int endchar, int n, int flags) +{ + PyObject *v; + int i; + int itemfailed = 0; + if (n < 0) + return NULL; + v = PyList_New(n); + if (v == NULL) + return NULL; + /* Note that we can't bail immediately on error as this will leak + refcounts on any 'N' arguments. */ + for (i = 0; i < n; i++) { + PyObject *w = do_mkvalue(p_format, p_va, flags); + if (w == NULL) { + itemfailed = 1; + Py_INCREF(Py_None); + w = Py_None; + } + PyList_SET_ITEM(v, i, w); + } + + if (itemfailed) { + /* do_mkvalue() should have already set an error */ + Py_DECREF(v); + return NULL; + } + if (**p_format != endchar) { + Py_DECREF(v); + PyErr_SetString(PyExc_SystemError, + "Unmatched paren in format"); + return NULL; + } + if (endchar) + ++*p_format; + return v; +} + +#ifdef Py_USING_UNICODE +static int +_ustrlen(Py_UNICODE *u) +{ + int i = 0; + Py_UNICODE *v = u; + while (*v != 0) { i++; v++; } + return i; +} +#endif + +static PyObject * +do_mktuple(const char **p_format, va_list *p_va, int endchar, int n, int flags) +{ + PyObject *v; + int i; + int itemfailed = 0; + if (n < 0) + return NULL; + if ((v = PyTuple_New(n)) == NULL) + return NULL; + /* Note that we can't bail immediately on error as this will leak + refcounts on any 'N' arguments. */ + for (i = 0; i < n; i++) { + PyObject *w = do_mkvalue(p_format, p_va, flags); + if (w == NULL) { + itemfailed = 1; + Py_INCREF(Py_None); + w = Py_None; + } + PyTuple_SET_ITEM(v, i, w); + } + if (itemfailed) { + /* do_mkvalue() should have already set an error */ + Py_DECREF(v); + return NULL; + } + if (**p_format != endchar) { + Py_DECREF(v); + PyErr_SetString(PyExc_SystemError, + "Unmatched paren in format"); + return NULL; + } + if (endchar) + ++*p_format; + return v; +} + +static PyObject * +do_mkvalue(const char **p_format, va_list *p_va, int flags) +{ + for (;;) { + switch (*(*p_format)++) { + case '(': + return do_mktuple(p_format, p_va, ')', + countformat(*p_format, ')'), flags); + + case '[': + return do_mklist(p_format, p_va, ']', + countformat(*p_format, ']'), flags); + + case '{': + return do_mkdict(p_format, p_va, '}', + countformat(*p_format, '}'), flags); + + case 'b': + case 'B': + case 'h': + case 'i': + return PyInt_FromLong((long)va_arg(*p_va, int)); + + case 'H': + return PyInt_FromLong((long)va_arg(*p_va, unsigned int)); + + case 'I': + { + Py_FatalError("I unsupported so far"); + //unsigned int n; + //n = va_arg(*p_va, unsigned int); + //if (n > (unsigned long)PyInt_GetMax()) + // return PyLong_FromUnsignedLong((unsigned long)n); + //else + // return PyInt_FromLong(n); + } + + case 'n': +#if SIZEOF_SIZE_T!=SIZEOF_LONG + return PyInt_FromSsize_t(va_arg(*p_va, Py_ssize_t)); +#endif + /* Fall through from 'n' to 'l' if Py_ssize_t is long */ + case 'l': + return PyInt_FromLong(va_arg(*p_va, long)); + + case 'k': + { + Py_FatalError("Py_BuildValue k unsupported so far\n"); + /* unsigned long n; */ + /* n = va_arg(*p_va, unsigned long); */ + /* if (n > (unsigned long)PyInt_GetMax()) */ + /* return PyLong_FromUnsignedLong(n); */ + /* else */ + /* return PyInt_FromLong(n); */ + } + +#ifdef HAVE_LONG_LONG + case 'L': + Py_FatalError("Py_BuildValue L unsupported for now\n"); + //return PyLong_FromLongLong((PY_LONG_LONG)va_arg(*p_va, PY_LONG_LONG)); + + case 'K': + Py_FatalError("Py_BuildValue K unsupported for now\n"); + //return PyLong_FromUnsignedLongLong((PY_LONG_LONG)va_arg(*p_va, unsigned PY_LONG_LONG)); +#endif +#ifdef Py_USING_UNICODE + case 'u': + { + PyObject *v; + Py_UNICODE *u = va_arg(*p_va, Py_UNICODE *); + Py_ssize_t n; + if (**p_format == '#') { + ++*p_format; + if (flags & FLAG_SIZE_T) + n = va_arg(*p_va, Py_ssize_t); + else + n = va_arg(*p_va, int); + } + else + n = -1; + if (u == NULL) { + v = Py_None; + Py_INCREF(v); + } + else { + if (n < 0) + n = _ustrlen(u); + v = PyUnicode_FromUnicode(u, n); + } + return v; + } +#endif + case 'f': + case 'd': + return PyFloat_FromDouble( + (double)va_arg(*p_va, va_double)); + +#ifndef WITHOUT_COMPLEX + case 'D': + return PyComplex_FromCComplex( + *((Py_complex *)va_arg(*p_va, Py_complex *))); +#endif /* WITHOUT_COMPLEX */ + + case 'c': + { + char p[1]; + p[0] = (char)va_arg(*p_va, int); + return PyString_FromStringAndSize(p, 1); + } + + case 's': + case 'z': + { + PyObject *v; + char *str = va_arg(*p_va, char *); + Py_ssize_t n; + if (**p_format == '#') { + ++*p_format; + if (flags & FLAG_SIZE_T) + n = va_arg(*p_va, Py_ssize_t); + else + n = va_arg(*p_va, int); + } + else + n = -1; + if (str == NULL) { + v = Py_None; + Py_INCREF(v); + } + else { + if (n < 0) { + size_t m = strlen(str); + if (m > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string too long for Python string"); + return NULL; + } + n = (Py_ssize_t)m; + } + v = PyString_FromStringAndSize(str, n); + } + return v; + } + + case 'N': + case 'S': + case 'O': + if (**p_format == '&') { + typedef PyObject *(*converter)(void *); + converter func = va_arg(*p_va, converter); + void *arg = va_arg(*p_va, void *); + ++*p_format; + return (*func)(arg); + } + else { + PyObject *v; + v = va_arg(*p_va, PyObject *); + if (v != NULL) { + if (*(*p_format - 1) != 'N') + Py_INCREF(v); + } + else if (!PyErr_Occurred()) + /* If a NULL was passed + * because a call that should + * have constructed a value + * failed, that's OK, and we + * pass the error on; but if + * no error occurred it's not + * clear that the caller knew + * what she was doing. */ + PyErr_SetString(PyExc_SystemError, + "NULL object passed to Py_BuildValue"); + return v; + } + + case ':': + case ',': + case ' ': + case '\t': + break; + + default: + PyErr_SetString(PyExc_SystemError, + "bad format char passed to Py_BuildValue"); + return NULL; + + } + } +} + + +PyObject * +Py_BuildValue(const char *format, ...) +{ + va_list va; + PyObject* retval; + va_start(va, format); + retval = va_build_value(format, va, 0); + va_end(va); + return retval; +} + +PyObject * +_Py_BuildValue_SizeT(const char *format, ...) +{ + va_list va; + PyObject* retval; + va_start(va, format); + retval = va_build_value(format, va, FLAG_SIZE_T); + va_end(va); + return retval; +} + +PyObject * +Py_VaBuildValue(const char *format, va_list va) +{ + return va_build_value(format, va, 0); +} + +PyObject * +_Py_VaBuildValue_SizeT(const char *format, va_list va) +{ + return va_build_value(format, va, FLAG_SIZE_T); +} + +static PyObject * +va_build_value(const char *format, va_list va, int flags) +{ + const char *f = format; + int n = countformat(f, '\0'); + va_list lva; + +#ifdef VA_LIST_IS_ARRAY + memcpy(lva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(lva, va); +#else + lva = va; +#endif +#endif + + if (n < 0) + return NULL; + if (n == 0) { + Py_INCREF(Py_None); + return Py_None; + } + if (n == 1) + return do_mkvalue(&f, &lva, flags); + return do_mktuple(&f, &lva, '\0', n, flags); +} + + +PyObject * +PyEval_CallFunction(PyObject *obj, const char *format, ...) +{ + va_list vargs; + PyObject *args; + PyObject *res; + + va_start(vargs, format); + + args = Py_VaBuildValue(format, vargs); + va_end(vargs); + + if (args == NULL) + return NULL; + + res = PyEval_CallObject(obj, args); + Py_DECREF(args); + + return res; +} + + +PyObject * +PyEval_CallMethod(PyObject *obj, const char *methodname, const char *format, ...) +{ + va_list vargs; + PyObject *meth; + PyObject *args; + PyObject *res; + + meth = PyObject_GetAttrString(obj, methodname); + if (meth == NULL) + return NULL; + + va_start(vargs, format); + + args = Py_VaBuildValue(format, vargs); + va_end(vargs); + + if (args == NULL) { + Py_DECREF(meth); + return NULL; + } + + res = PyEval_CallObject(meth, args); + Py_DECREF(meth); + Py_DECREF(args); + + return res; +} int PyModule_AddObject(PyObject *m, const char *name, PyObject *o) @@ -29,3 +547,26 @@ return 0; } +int +PyModule_AddIntConstant(PyObject *m, const char *name, long value) +{ + PyObject *o = PyInt_FromLong(value); + if (!o) + return -1; + if (PyModule_AddObject(m, name, o) == 0) + return 0; + Py_DECREF(o); + return -1; +} + +int +PyModule_AddStringConstant(PyObject *m, const char *name, const char *value) +{ + PyObject *o = PyString_FromString(value); + if (!o) + return -1; + if (PyModule_AddObject(m, name, o) == 0) + return 0; + Py_DECREF(o); + return -1; +} From fijal at codespeak.net Mon Apr 5 05:35:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 05:35:22 +0200 (CEST) Subject: [pypy-svn] r73408 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405033522.34BFE282B90@codespeak.net> Author: fijal Date: Mon Apr 5 05:35:20 2010 New Revision: 73408 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: explode in a more reasonable manner Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Mon Apr 5 05:35:20 2010 @@ -30,4 +30,6 @@ space.int_w(space.getitem(w_obj, space.wrap(i)))) elif c == ':': return 1 + else: + raise Exception("Unsupported parameter: %s" % (c,)) i += 1 From afa at codespeak.net Mon Apr 5 14:04:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 5 Apr 2010 14:04:54 +0200 (CEST) Subject: [pypy-svn] r73409 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405120454.8A803282B90@codespeak.net> Author: afa Date: Mon Apr 5 14:04:51 2010 New Revision: 73409 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Log: Remove outdated comment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Mon Apr 5 14:04:51 2010 @@ -6,6 +6,4 @@ def PyMapping_Keys(space, w_obj): """On success, return a list of the keys in object o. On failure, return NULL. This is equivalent to the Python expression o.keys().""" - # XXX: Cpython implements this in terms of PyObject_CallMethod, we should - # do that eventually. return space.call_method(w_obj, "keys") From trundle at codespeak.net Mon Apr 5 15:30:44 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 15:30:44 +0200 (CEST) Subject: [pypy-svn] r73410 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100405133044.8DD71282B90@codespeak.net> Author: trundle Date: Mon Apr 5 15:30:42 2010 New Revision: 73410 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_AS_DATA and PyUnicode_GET_DATA_SIZE. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 15:30:42 2010 @@ -274,6 +274,12 @@ (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) +PyUnicodeObjectStruct = lltype.ForwardReference() +PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) +PyUnicodeObjectFields = (PyObjectFields + + (("buffer", rffi.VOIDP), ("size", Py_ssize_t))) +cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) + VA_TP_LIST = {'int': lltype.Signed, 'PyObject*': PyObject, 'int*': rffi.INTP} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Mon Apr 5 15:30:42 2010 @@ -1,9 +1,26 @@ +#ifndef Py_UNICODEOBJECT_H +#define Py_UNICODEOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct { PyObject_HEAD + void *buffer; + Py_ssize_t size; } PyUnicodeObject; + + //XXX typedef unsigned int Py_UCS4; // pypy only supports only UCS4 #define PY_UNICODE_TYPE Py_UCS4 typedef PY_UNICODE_TYPE Py_UNICODE; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_UNICODEOBJECT_H */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Mon Apr 5 15:30:42 2010 @@ -3,9 +3,10 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, PyObject, PyStringObject, ADDR,\ - Py_TPFLAGS_HEAPTYPE + Py_TPFLAGS_HEAPTYPE, PyUnicodeObject from pypy.module.cpyext.state import State from pypy.objspace.std.stringobject import W_StringObject +from pypy.objspace.std.unicodeobject import W_UnicodeObject from pypy.rlib.objectmodel import we_are_translated #________________________________________________________ @@ -73,6 +74,14 @@ py_obj = rffi.cast(PyObject, py_obj_str) py_obj.c_ob_refcnt = 1 py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) + elif isinstance(w_obj, W_UnicodeObject): + py_obj_unicode = lltype.malloc(PyUnicodeObject.TO, flavor='raw', zero=True) + py_obj_unicode.c_size = len(space.unicode_w(w_obj)) + py_obj_unicode.c_buffer = lltype.nullptr(rffi.VOIDP.TO) + pto = make_ref(space, space.w_unicode) + py_obj = rffi.cast(PyObject, py_obj_unicode) + py_obj.c_ob_refcnt = 1 + py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) else: py_obj = lltype.malloc(PyObject.TO, flavor="raw", zero=True) py_obj.c_ob_refcnt = 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 5 15:30:42 2010 @@ -5653,27 +5653,12 @@ """ raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyUnicode_GET_DATA_SIZE(space, o): - """Return the size of the object's internal buffer in bytes. o has to be a - PyUnicodeObject (not checked). - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject], {Py_UNICODE*}) def PyUnicode_AS_UNICODE(space, o): """Return a pointer to the internal Py_UNICODE buffer of the object. o has to be a PyUnicodeObject (not checked).""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) -def PyUnicode_AS_DATA(space, o): - """Return a pointer to the internal buffer of the object. o has to be a - PyUnicodeObject (not checked).""" - raise NotImplementedError - @cpython_api([], rffi.INT_real) def PyUnicode_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Mon Apr 5 15:30:42 2010 @@ -1,10 +1,18 @@ # encoding: iso-8859-15 from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.unicodeobject import Py_UNICODE +from pypy.rpython.lltypesystem import rffi class TestUnicode(BaseApiTest): def test_unicodeobject(self, space, api): assert space.unwrap(api.PyUnicode_GET_SIZE(space.wrap(u'sp?m'))) == 4 + def test_AS_DATA(self, space, api): + word = space.wrap(u'spam') + array = rffi.cast(rffi.CWCHARP, api.PyUnicode_AS_DATA(word)) + for (i, char) in enumerate(space.unwrap(word)): + assert array[i] == char + def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Mon Apr 5 15:30:42 2010 @@ -1,8 +1,8 @@ -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb -from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, +from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, build_type_checkers, cpython_api) -from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.pyobject import PyObject, from_ref from pypy.objspace.std import unicodeobject PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode") @@ -55,6 +55,24 @@ """Return the character ch converted to lower case.""" return unicodedb.tolower(w_ch) + at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) +def PyUnicode_AS_DATA(space, ref): + """Return a pointer to the internal buffer of the object. o has to be a + PyUnicodeObject (not checked).""" + ref_unicode = rffi.cast(PyUnicodeObject, ref) + if not ref_unicode.c_buffer: + # Copy unicode buffer + w_unicode = from_ref(space, ref) + u = space.unicode_w(w_unicode) + ref_unicode.c_buffer = rffi.cast(rffi.VOIDP, rffi.unicode2wcharp(u)) + return rffi.cast(rffi.CCHARP, ref_unicode.c_buffer) + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyUnicode_GET_DATA_SIZE(space, obj): + """Return the size of the object's internal buffer in bytes. o has to be a + PyUnicodeObject (not checked).""" + return rffi.sizeof(lltype.UniChar) * (PyUnicode_GET_SIZE(space, obj) + 1) + @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyUnicode_GET_SIZE(space, w_obj): """Return the size of the object. o has to be a PyUnicodeObject (not From trundle at codespeak.net Mon Apr 5 16:08:45 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Mon, 5 Apr 2010 16:08:45 +0200 (CEST) Subject: [pypy-svn] r73411 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100405140845.E13C1282B90@codespeak.net> Author: trundle Date: Mon Apr 5 16:08:44 2010 New Revision: 73411 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Fix PyUnicode_GET_DATA_SIZE and PyUnicode_GET_SIZE. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Mon Apr 5 16:08:44 2010 @@ -5,7 +5,9 @@ class TestUnicode(BaseApiTest): def test_unicodeobject(self, space, api): - assert space.unwrap(api.PyUnicode_GET_SIZE(space.wrap(u'sp?m'))) == 4 + assert api.PyUnicode_GET_SIZE(space.wrap(u'sp?m')) == 4 + # XXX assuming UCS-4 + assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 16 def test_AS_DATA(self, space, api): word = space.wrap(u'spam') Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Mon Apr 5 16:08:44 2010 @@ -71,7 +71,7 @@ def PyUnicode_GET_DATA_SIZE(space, obj): """Return the size of the object's internal buffer in bytes. o has to be a PyUnicodeObject (not checked).""" - return rffi.sizeof(lltype.UniChar) * (PyUnicode_GET_SIZE(space, obj) + 1) + return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, obj) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyUnicode_GET_SIZE(space, w_obj): @@ -81,4 +81,4 @@ This function returned an int type. This might require changes in your code for properly supporting 64-bit systems.""" assert isinstance(w_obj, unicodeobject.W_UnicodeObject) - return space.len(w_obj) + return space.int_w(space.len(w_obj)) From xoraxax at codespeak.net Mon Apr 5 17:33:18 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 17:33:18 +0200 (CEST) Subject: [pypy-svn] r73412 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100405153318.34656282B90@codespeak.net> Author: xoraxax Date: Mon Apr 5 17:33:14 2010 New Revision: 73412 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Refactor the leak checking code to work also with api tests. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py Mon Apr 5 17:33:14 2010 @@ -3,6 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.state import State from pypy.module.cpyext import api +from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, check_and_print_leaks PyObject = api.PyObject @api.cpython_api([PyObject], lltype.Void) @@ -21,9 +22,14 @@ cls.api = CAPI() CAPI.__dict__.update(api.INTERPLEVEL_API) + def setup_method(self, func): + freeze_refcnts(self) + def teardown_method(self, func): state = self.space.fromcache(State) assert state.exc_value is None + if check_and_print_leaks(self): + assert False, "Test leaks or loses object(s)." class TestConversion(BaseApiTest): def test_conversions(self, space, api): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Mon Apr 5 17:33:14 2010 @@ -58,6 +58,65 @@ standalone=False) return str(soname) +def freeze_refcnts(self): + state = self.space.fromcache(State) + self.frozen_refcounts = {} + for w_obj, obj in state.py_objects_w2r.iteritems(): + self.frozen_refcounts[w_obj] = obj.c_ob_refcnt + #state.print_refcounts() + self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) + self.frozen_lltallocations = lltype.ALLOCATED.copy() + lltype.TRACK_ALLOCATIONS = True + +def check_and_print_leaks(self): + # check for sane refcnts + leaking = False + state = self.space.fromcache(State) + import gc + gc.collect() + lost_objects_w = identity_dict() + lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) + for w_obj, obj in state.py_objects_w2r.iteritems(): + base_refcnt = self.frozen_refcounts.get(w_obj) + delta = obj.c_ob_refcnt + if base_refcnt is not None: + delta -= base_refcnt + lost_objects_w.pop(w_obj) + if delta != 0: + leaking = True + print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) + lifeline = w_obj.get_pyolifeline() + if lifeline is not None: + refcnt = lifeline.pyo.c_ob_refcnt + if refcnt > 0: + print >>sys.stderr, "\tThe object also held by C code." + else: + referrers_repr = [] + for o in gc.get_referrers(w_obj): + try: + repr_str = repr(o) + except TypeError, e: + repr_str = "%s (type of o is %s)" % (str(e), type(o)) + referrers_repr.append(repr_str) + referrers = ", ".join(referrers_repr) + print >>sys.stderr, "\tThe object is referenced by these objects:", \ + referrers + for w_obj in lost_objects_w: + print >>sys.stderr, "Lost object %r" % (w_obj, ) + leaking = True + for llvalue in set(ll2ctypes.ALLOCATED.values()) - self.frozen_ll2callocations: + if getattr(llvalue, "_traceback", None): # this means that the allocation should be tracked + leaking = True + print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) + print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) + for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): + leaking = True + print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) + print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) + + return leaking + + class AppTestCpythonExtensionBase: def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext']) @@ -123,7 +182,7 @@ def setup_method(self, func): self.w_import_module = self.space.wrap(self.import_module) self.w_import_extension = self.space.wrap(self.import_extension) - self.freeze_refcnts() + freeze_refcnts(self) #self.check_and_print_leaks() def teardown_method(self, func): @@ -140,68 +199,9 @@ pass except AttributeError: pass - state = self.space.fromcache(State) - if self.check_and_print_leaks(): + if check_and_print_leaks(self): assert False, "Test leaks or loses object(s)." - def freeze_refcnts(self): - state = self.space.fromcache(State) - self.frozen_refcounts = {} - for w_obj, obj in state.py_objects_w2r.iteritems(): - self.frozen_refcounts[w_obj] = obj.c_ob_refcnt - #state.print_refcounts() - self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) - self.frozen_lltallocations = lltype.ALLOCATED.copy() - lltype.TRACK_ALLOCATIONS = True - - def check_and_print_leaks(self): - # check for sane refcnts - leaking = False - state = self.space.fromcache(State) - import gc - gc.collect() - lost_objects_w = identity_dict() - lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - for w_obj, obj in state.py_objects_w2r.iteritems(): - base_refcnt = self.frozen_refcounts.get(w_obj) - delta = obj.c_ob_refcnt - if base_refcnt is not None: - delta -= base_refcnt - lost_objects_w.pop(w_obj) - if delta != 0: - leaking = True - print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = w_obj.get_pyolifeline() - if lifeline is not None: - refcnt = lifeline.pyo.c_ob_refcnt - if refcnt > 0: - print >>sys.stderr, "\tThe object also held by C code." - else: - referrers_repr = [] - for o in gc.get_referrers(w_obj): - try: - repr_str = repr(o) - except TypeError, e: - repr_str = "%s (type of o is %s)" % (str(e), type(o)) - referrers_repr.append(repr_str) - referrers = ", ".join(referrers_repr) - print >>sys.stderr, "\tThe object is referenced by these objects:", \ - referrers - for w_obj in lost_objects_w: - print >>sys.stderr, "Lost object %r" % (w_obj, ) - leaking = True - for llvalue in set(ll2ctypes.ALLOCATED.values()) - self.frozen_ll2callocations: - if getattr(llvalue, "_traceback", None): # this means that the allocation should be tracked - leaking = True - print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) - print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): - leaking = True - print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) - print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - - return leaking - class AppTestCpythonExtension(AppTestCpythonExtensionBase): def test_createmodule(self): From xoraxax at codespeak.net Mon Apr 5 17:41:35 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 17:41:35 +0200 (CEST) Subject: [pypy-svn] r73413 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100405154135.51152282B90@codespeak.net> Author: xoraxax Date: Mon Apr 5 17:41:33 2010 New Revision: 73413 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix leaks in test_object and test_unicodeobject. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Mon Apr 5 17:41:33 2010 @@ -1,7 +1,7 @@ import py from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, lltype class TestObject(BaseApiTest): @@ -44,10 +44,14 @@ assert x.test == 10 def test_getattr_string(self, space, api): - assert api.PyObject_GetAttrString(space.wrap(""), rffi.str2charp("__len__")) - assert not api.PyObject_GetAttrString(space.wrap(""), rffi.str2charp("not_real")) + charp1 = rffi.str2charp("__len__") + charp2 = rffi.str2charp("not_real") + assert api.PyObject_GetAttrString(space.wrap(""), charp1) + assert not api.PyObject_GetAttrString(space.wrap(""), charp2) assert api.PyErr_Occurred() is space.w_AttributeError api.PyErr_Clear() + lltype.free(charp1, flavor="raw") + lltype.free(charp2, flavor="raw") def test_getitem(self, space, api): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Mon Apr 5 17:41:33 2010 @@ -15,7 +15,8 @@ from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ - Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS + Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ + PyUnicodeObject from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod @@ -319,6 +320,17 @@ Py_DecRef(space, pto) @cpython_api([PyObject], lltype.Void, external=False) +def unicode_dealloc(space, obj): + obj = rffi.cast(PyUnicodeObject, obj) + pto = obj.c_ob_type + if obj.c_buffer: + lltype.free(obj.c_buffer, flavor="raw") + obj_voidp = rffi.cast(rffi.VOIDP_real, obj) + generic_cpy_call(space, pto.c_tp_free, obj_voidp) + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) + + at cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) @@ -357,6 +369,9 @@ elif space.is_w(w_type, space.w_str): pto.c_tp_dealloc = llhelper(string_dealloc.api_func.functype, string_dealloc.api_func.get_wrapper(space)) + elif space.is_w(w_type, space.w_unicode): + pto.c_tp_dealloc = llhelper(unicode_dealloc.api_func.functype, + unicode_dealloc.api_func.get_wrapper(space)) else: pto.c_tp_dealloc = llhelper(subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) From fijal at codespeak.net Mon Apr 5 18:00:06 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 18:00:06 +0200 (CEST) Subject: [pypy-svn] r73414 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100405160006.A2547282B90@codespeak.net> Author: fijal Date: Mon Apr 5 18:00:05 2010 New Revision: 73414 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Silly, fix the test Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Mon Apr 5 18:00:05 2010 @@ -3,6 +3,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.stringobject import new_empty_str from pypy.module.cpyext.api import PyStringObject, PyObjectP, PyObject +from pypy.module.cpyext.pyobject import Py_DecRef import py import sys @@ -146,5 +147,5 @@ assert py_str.c_size == 10 assert py_str.c_buffer[1] == 'b' assert py_str.c_buffer[10] == '\x00' + Py_DecRef(space, ar[0]) lltype.free(ar, flavor='raw') - From iammisc at codespeak.net Mon Apr 5 18:48:40 2010 From: iammisc at codespeak.net (iammisc at codespeak.net) Date: Mon, 5 Apr 2010 18:48:40 +0200 (CEST) Subject: [pypy-svn] r73415 - in pypy/branch/jit-stackless/pypy/jit: backend/llgraph backend/x86 backend/x86/test metainterp/test Message-ID: <20100405164840.D5480282B90@codespeak.net> Author: iammisc Date: Mon Apr 5 18:48:39 2010 New Revision: 73415 Added: pypy/branch/jit-stackless/pypy/jit/backend/x86/test/test_stackless_integration.py Modified: pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py Log: Added some tests for x86 Modified: pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/llgraph/llimpl.py Mon Apr 5 18:48:39 2010 @@ -463,6 +463,10 @@ log.trace('failed: %s' % ( ', '.join(map(str, fail_args)),)) return op.fail_index + except UnwindException: + assert op.is_guard() + self._populate_fail_args(op) + raise #verbose = self.verbose assert (result is None) == (op.result is None) if op.result is not None: Modified: pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-stackless/pypy/jit/backend/x86/assembler.py Mon Apr 5 18:48:39 2010 @@ -32,11 +32,11 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) -JIT_STATE_HEADER = lltype.Struct('JIT_STATE_HEADER', - ('header', STATE_HEADER), - ('saved_ints', lltype.Ptr(lltype.GcArray(lltype.Signed))), - ('saved_floats', lltype.Ptr(lltype.GcArray(lltype.Float))), - ('saved_refs', lltype.Ptr(lltype.GcArray(lltype.Ptr)))) +JIT_STATE_HEADER = lltype.GcStruct('JIT_STATE_HEADER', + ('header', STATE_HEADER), + ('saved_ints', lltype.Ptr(lltype.GcArray(lltype.Signed))), + ('saved_floats', lltype.Ptr(lltype.GcArray(lltype.Float))), + ('saved_refs', lltype.Ptr(lltype.GcArray(llmemory.GCREF)))) class MachineCodeBlockWrapper(object): MC_DEFAULT_SIZE = 1024*1024 @@ -987,7 +987,7 @@ loc1 = locs[1] if self.cpu.stackless: unwinder_addr = self.generate_unwinder_failure(guard_op.descr, guard_op.fail_args, faillocs) - stku_type, stku_instance = self.cpu.get_unwind_exception() + stku_type, stku_instance = self.cpu.get_unwind_exception() self.mc.CMP(heap(self.cpu.pos_exception()), imm(stku_type)) self.mc.JE(rel32(unwinder_addr)) # if this is indeed an unwind exception jump to saver code self.mc.MOV(loc1, heap(self.cpu.pos_exception())) @@ -1228,37 +1228,97 @@ @rgc.no_collect def unwind_frame_values(self, bytecode, frame_addr, allregisters): - # Okay allocate a frame for us, + # Allocate a frame for us, - # then set the proper values in global_state + # XXX set the proper values in global_state + self.fail_ebp = allregisters[16 + ebp.op] - bytecode = rffi.cast(rffi.UCHARP, bytecode) boxes = [] + kinds = [] + ints, floats, refs = (0, 0, 0) + int_num, float_num, ref_num = (0, 0, 0) + int_boxes, float_boxes, ref_boxes = (None, None, None) + num = 0 while 1: # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - kind = code & 3 - while code > 0x7F: - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - index = len(boxes) - # Now kind contains information at the type of data that will go here - if kind != self.DESCR_SPECIAL: - kinds.append(kind) + if code >= self.CODE_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code - self.CODE_FROMSTACK) >> 2 + stackloc = frame_addr + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + kinds.append((kind, True, stackloc)) else: - if code == self.CODE_STOP: + kind = code & 3 + if kind == self.DESCR_SPECIAL: + if code == self.CODE_HOLE: + num += 1 + continue + assert code == self.CODE_STOP break - assert code == self.CODE_HOLE, "Bad code" + code >>= 2 + kinds.append((kind, False, code)) + + if kind == self.DESCR_INT: + ints += 1 + elif kind == self.DESCR_FLOAT: + floats += 1 + elif kind == self.DESCR_REF: + refs += 1 - # Now kinds contains the types we need to store + # Now kinds contains the types we need to store in the right order state_header = lltype.malloc(JIT_STATE_HEADER, zero=True) - state_header.saved_ints = lltype.malloc(lltype.GcArray(lltype.Signed), len([x for x in kinds if x == self.DESCR_INT])) - state_header.saved_floats = lltype.malloc(lltype.GcArray(lltype.Float), len([x for x in kinds if x == self.DESCR_FLOAT])) - state_header.saved_floats = lltype.malloc(lltype.GcArray(lltype.Ptr), len([x for x in kinds if x == self.DESCR_REF])) + if ints: + int_boxes = values_array(lltype.Signed, ints) + state_header.saved_ints = int_boxes.ar + if floats: + float_boxes = values_array(lltype.Float, floats) + state_header.saved_floats = float_boxes.ar + if refs: + ref_boxes = values_array(llmemory.GCREF, refs) + state_header.saved_refs = ref_boxes.ar + + value, value_hi = (0, 0) + + for (kind, in_stack, loc) in kinds: + if in_stack: + value = rffi.cast(rffi.LONGP, loc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + value = rffi.cast(rffi.LONGP, loc - 4)[0] + else: + if kind == self.DESCR_FLOAT: + value = allregisters[2*loc] + value_hi = allregisters[2*loc+1] + else: + value = allregisters[16 + loc] - + if kind == self.DESCR_INT: + tgt = int_boxes.get_addr_for_num(int_num) + int_num += 1 + elif kind == self.DESCR_REF: + tgt = ref_boxes.get_addr_for_num(ref_num) + ref_num += 1 + elif kind == self.DESCR_FLOAT: + tgt = float_boxes.get_addr_for_num(float_num) + float_num += 1 + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + rffi.cast(rffi.LONGP, tgt)[0] = value + + return state_header @rgc.no_collect def grab_frame_values(self, bytecode, frame_addr, allregisters): Added: pypy/branch/jit-stackless/pypy/jit/backend/x86/test/test_stackless_integration.py ============================================================================== --- (empty file) +++ pypy/branch/jit-stackless/pypy/jit/backend/x86/test/test_stackless_integration.py Mon Apr 5 18:48:39 2010 @@ -0,0 +1,141 @@ +from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.assembler import Assembler386 +from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory +from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs +from pypy.jit.backend.x86.test.test_assembler import FakeMC, FakeCPU + +def do_unwinder_recovery_func(withfloats=False): + import random + S = lltype.GcStruct('S') + + def get_random_int(): + return random.randrange(-10000, 10000) + def get_random_ptr(): + return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) + + def get_random_float(): + assert withfloats + value = random.random() - 0.5 + # make sure it fits into 64 bits + tmp = lltype.malloc(rffi.LONGP.TO, 2, flavor='raw') + rffi.cast(rffi.DOUBLEP, tmp)[0] = value + return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] + + # memory locations: 26 integers, 26 pointers, 26 floats + # main registers: half of them as signed and the other half as ptrs + # xmm registers: all floats, from xmm0 to xmm7 + # holes: 8 + locations = [] + baseloc = 4 + for i in range(26+26+26): + if baseloc < 128: + baseloc += random.randrange(2, 20) + else: + baseloc += random.randrange(2, 1000) + locations.append(baseloc) + random.shuffle(locations) + content = ([('int', locations.pop()) for _ in range(26)] + + [('ptr', locations.pop()) for _ in range(26)] + + [(['int', 'ptr'][random.randrange(0, 2)], reg) + for reg in [eax, ecx, edx, ebx, esi, edi]]) + if withfloats: + content += ([('float', locations.pop()) for _ in range(26)] + + [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7]]) + for i in range(8): + content.append(('hole', None)) + random.shuffle(content) + + # prepare the expected target arrays, the descr_bytecode, + # the 'registers' and the 'stack' arrays according to 'content' + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + registers = rffi.ptradd(xmmregisters, 16) + stacklen = baseloc + 10 + stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') + expected_ints = [] + expected_ptrs = [] + expected_floats = [] + + def write_in_stack(loc, value): + assert loc >= 0 + ofs = get_ebp_ofs(loc) + assert ofs < 0 + assert (ofs % 4) == 0 + stack[stacklen + ofs//4] = value + + descr_bytecode = [] + for i, (kind, loc) in enumerate(content): + if kind == 'hole': + num = Assembler386.CODE_HOLE + else: + if kind == 'float': + value, lo, hi = get_random_float() + expected_floats.append(value) + kind = Assembler386.DESCR_FLOAT + if isinstance(loc, REG): + xmmregisters[2*loc.op] = lo + xmmregisters[2*loc.op+1] = hi + else: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) + else: + if kind == 'int': + value = get_random_int() + expected_ints.append(value) + kind = Assembler386.DESCR_INT + elif kind == 'ptr': + value = get_random_ptr() + expected_ptrs.append(value) + kind = Assembler386.DESCR_REF + value = rffi.cast(rffi.LONG, value) + else: + assert 0, kind + if isinstance(loc, REG): + registers[loc.op] = value + else: + write_in_stack(loc, value) + + if isinstance(loc, REG): + num = kind + 4*loc.op + else: + num = kind + 4*(8+loc) + while num >= 0x80: + descr_bytecode.append((num & 0x7F) | 0x80) + num >>= 7 + descr_bytecode.append(num) + + descr_bytecode.append(Assembler386.CODE_STOP) + descr_bytecode.append(0xC3) # fail_index = 0x1C3 + descr_bytecode.append(0x01) + descr_bytecode.append(0x00) + descr_bytecode.append(0x00) + descr_bytecode.append(0xCC) # end marker + descr_bytes = lltype.malloc(rffi.UCHARP.TO, len(descr_bytecode), + flavor='raw') + for i in range(len(descr_bytecode)): + assert 0 <= descr_bytecode[i] <= 255 + descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) + registers[8] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen + + # test + assembler = Assembler386(FakeCPU()) + state_header = assembler.unwinder_recovery_func(registers) + + if state_header.saved_ints: + for x,y in zip(state_header.saved_ints, expected_ints): + assert x == y + + if state_header.saved_floats: + for x,y in zip(state_header.saved_floats, expected_floats): + assert x == y + + if state_header.saved_refs: + for x,y in zip(state_header.saved_refs, expected_ptrs): + assert x == y + +def test_unwinder_recovery(): + do_unwinder_recovery_func() + +def test_unwinder_recovery_with_floats(): + do_unwinder_recovery_func(withfloats=True) Modified: pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py ============================================================================== --- pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py (original) +++ pypy/branch/jit-stackless/pypy/jit/metainterp/test/test_jit_stackless_interop.py Mon Apr 5 18:48:39 2010 @@ -13,8 +13,8 @@ from pypy.translator.stackless.code import yield_current_frame_to_caller from pypy.translator.translator import TranslationContext, graphof from pypy.annotation.listdef import s_list_of_strings +from pypy.rpython import llinterp from pypy.translator.stackless.transform import StacklessTransformer, FrameTyper -from pypy.translator.c.genc import CStandaloneBuilder from pypy.translator.c import gc from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache from pypy import conftest @@ -57,3 +57,42 @@ return x res = meta_interp(f, [100]) assert res == 100 + +def rtype_stackless_function(fn): + t = TranslationContext() + t.config.translation.stackless = True + annotator = t.buildannotator() + annotator.policy.allow_someobjects = False + + s_returnvar = annotator.build_types(fn, [s_list_of_strings]) + if not isinstance(s_returnvar, annmodel.SomeInteger): + raise Exception, "this probably isn't going to work" + t.buildrtyper().specialize() + + from pypy.translator.transform import insert_ll_stackcheck + insert_ll_stackcheck(t) + +# if conftest.option.view: +# t.view() + return t + +def llinterp_stackless_function(fn, returntranslator=False, + assert_unwind=True): + def wrapper(argv): + return fn() + t = rtype_stackless_function(wrapper) + st = StacklessTransformer(t, wrapper, assert_unwind=assert_unwind) + st.transform_all() + if conftest.option.view: + t.view() + + graph = graphof(t, st.slp_entry_point) + r_list_of_strings = t.rtyper.getrepr( + t.annotator.binding(graph.startblock.inputargs[0])) + ll_list = r_list_of_strings.convert_const(['']) + interp = llinterp.LLInterpreter(t.rtyper) + res = interp.eval_graph(graph, [ll_list]) + if returntranslator: + return res, t + else: + return res From fijal at codespeak.net Mon Apr 5 21:38:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 21:38:29 +0200 (CEST) Subject: [pypy-svn] r73416 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100405193829.0B6B8282B9D@codespeak.net> Author: fijal Date: Mon Apr 5 21:38:27 2010 New Revision: 73416 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: (trundle, fijal) try to implement 'O' Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 5 21:38:27 2010 @@ -282,6 +282,7 @@ VA_TP_LIST = {'int': lltype.Signed, 'PyObject*': PyObject, + 'PyObject**': PyObjectP, 'int*': rffi.INTP} def configure_types(): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Mon Apr 5 21:38:27 2010 @@ -1,7 +1,9 @@ - +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL, \ VA_LIST_P, cpython_api_c from pypy.module.cpyext import api +from pypy.module.cpyext.pyobject import from_ref, make_ref,\ + add_borrowed_object, register_container from pypy.rpython.lltypesystem import lltype, rffi @cpython_api_c() @@ -19,17 +21,31 @@ @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=0) def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): - i = 0 + arg_i = 0 + fmt_i = 0 while True: - c = fmt[i] + c = fmt[fmt_i] if c == "\x00": return 1 elif c == "i": arr = api.va_get_int_star(va_list_p) arr[0] = rffi.cast(rffi.INT, - space.int_w(space.getitem(w_obj, space.wrap(i)))) + space.int_w(space.getitem(w_obj, space.wrap(arg_i)))) + elif c == "O": + w_item = space.getitem(w_obj, space.wrap(arg_i)) + if fmt[fmt_i + 1] == "!": + fmt_i += 1 + w_type = from_ref(space, api.va_get_PyObject_star(va_list_p)) + if not space.is_true(space.isinstance(w_item, w_type)): + raise OperationError(space.w_TypeError, + space.wrap("wrong type")) + arr = api.va_get_PyObject_star_star(va_list_p) + arr[0] = make_ref(space, w_item) + register_container(space, w_obj) + add_borrowed_object(space, arr[0]) elif c == ':': return 1 else: raise Exception("Unsupported parameter: %s" % (c,)) - i += 1 + arg_i += 1 + fmt_i += 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Mon Apr 5 21:38:27 2010 @@ -21,6 +21,24 @@ return NULL; } return PyInt_FromLong(l); + '''), + ('oneargobject', 'METH_VARARGS', + ''' + PyObject *obj; + if (!PyArg_Parse(args, "O", &obj)) { + return NULL; + } + Py_INCREF(obj); + return obj; + '''), + ('oneargobjectandlisttype', 'METH_VARARGS', + ''' + PyObject *obj; + if (!PyArg_Parse(args, "O!", &PyList_Type, &obj)) { + return NULL; + } + Py_INCREF(obj); + return obj; ''')]) assert mod.oneargint(1) == 1 raises(TypeError, mod.oneargint, None) @@ -31,4 +49,8 @@ else: raise Exception("DID NOT RAISE") assert mod.oneargandform(1) == 1 - + + sentinel = object() + res = mod.oneargobject(sentinel) + raises(TypeError, "mod.oneargobjectandlisttype(sentinel)") + assert res is sentinel From fijal at codespeak.net Mon Apr 5 22:26:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 22:26:33 +0200 (CEST) Subject: [pypy-svn] r73417 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405202633.BD348282B9E@codespeak.net> Author: fijal Date: Mon Apr 5 22:26:32 2010 New Revision: 73417 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Log: I think this is needed here, does not fix the test though Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Mon Apr 5 22:26:32 2010 @@ -40,7 +40,7 @@ raise OperationError(space.w_TypeError, space.wrap("wrong type")) arr = api.va_get_PyObject_star_star(va_list_p) - arr[0] = make_ref(space, w_item) + arr[0] = make_ref(space, w_item, borrowed=True) register_container(space, w_obj) add_borrowed_object(space, arr[0]) elif c == ':': From fijal at codespeak.net Mon Apr 5 22:47:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 22:47:48 +0200 (CEST) Subject: [pypy-svn] r73418 - pypy/extradoc/talk/oopsla2010 Message-ID: <20100405204748.3C87C282B9D@codespeak.net> Author: fijal Date: Mon Apr 5 22:47:46 2010 New Revision: 73418 Modified: pypy/extradoc/talk/oopsla2010/paper.txt Log: Provide a py.test traceback example Modified: pypy/extradoc/talk/oopsla2010/paper.txt ============================================================================== --- pypy/extradoc/talk/oopsla2010/paper.txt (original) +++ pypy/extradoc/talk/oopsla2010/paper.txt Mon Apr 5 22:47:46 2010 @@ -22,7 +22,24 @@ idea is found with web frameworks written in python - they usually try to present more information than classic traceback representation. -XXX example of django traceback +Example of a py.test traceback, using local variable introspection to find out +what happened: + +=================================== FAILURES =================================== +___________________________________ test_one ___________________________________ + + def test_one(): +> f(0, 0) + +test_x.py:6: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +i = 0, j = 0 + + def f(i, j): +> return i / j +E ZeroDivisionError: integer division or modulo by zero + In this paper we present the way we deal with frames in order to completely (or mostly) avoid costs associated with them. Our approach is giving From xoraxax at codespeak.net Mon Apr 5 22:56:10 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 22:56:10 +0200 (CEST) Subject: [pypy-svn] r73419 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405205610.AD2EE282B9C@codespeak.net> Author: xoraxax Date: Mon Apr 5 22:56:08 2010 New Revision: 73419 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix refcounting issue that showed up after running fuu2(u"uuu") in the following tests. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Mon Apr 5 22:56:08 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - Py_ssize_t, PyVarObject + Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State @@ -27,6 +27,8 @@ pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) obj_voidp = rffi.cast(rffi.VOIDP_real, obj) generic_cpy_call(space, pto.c_tp_free, obj_voidp) + if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, rffi.cast(PyObject, obj.c_ob_type)) @cpython_api([PyObject], rffi.INT_real, error=-1) def PyObject_IsTrue(space, w_obj): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Mon Apr 5 22:56:08 2010 @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, PyObject, PyStringObject, ADDR,\ - Py_TPFLAGS_HEAPTYPE, PyUnicodeObject + Py_TPFLAGS_HEAPTYPE, PyUnicodeObject, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.unicodeobject import W_UnicodeObject @@ -18,7 +18,7 @@ class InvalidPointerException(Exception): pass -DEBUG_REFCOUNT = False +DEBUG_REFCOUNT = True def debug_refcount(*args, **kwargs): frame_stackdepth = kwargs.pop("frame_stackdepth", 2) @@ -33,7 +33,6 @@ def make_ref(space, w_obj, borrowed=False, steal=False): from pypy.module.cpyext.typeobject import allocate_type_obj,\ W_PyCTypeObject, PyOLifeline - from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) @@ -150,11 +149,12 @@ if obj.c_ob_refcnt == 0: state = space.fromcache(State) ptr = rffi.cast(ADDR, obj) - if ptr not in state.py_objects_r2w and \ - space.is_w(from_ref(space, rffi.cast(PyObject, obj.c_ob_type)), space.w_str): - # this is a half-allocated string, lets call the deallocator - # without modifying the r2w/w2r dicts - _Py_Dealloc(space, obj) + if ptr not in state.py_objects_r2w: + w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): + # this is a half-allocated string, lets call the deallocator + # without modifying the r2w/w2r dicts + _Py_Dealloc(space, obj) else: w_obj = state.py_objects_r2w[ptr] del state.py_objects_r2w[ptr] @@ -180,7 +180,9 @@ hex(containee) del state.borrow_mapping[ptr] else: - assert obj.c_ob_refcnt > 0 + if not we_are_translated() and obj.c_ob_refcnt < 0: + print >>sys.stderr, "Negative refcount for obj %s with type %s" % (obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) + assert False @cpython_api([PyObject], lltype.Void) def Py_IncRef(space, obj): @@ -192,7 +194,6 @@ debug_refcount("INCREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) def _Py_Dealloc(space, obj): - from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.api import generic_cpy_call_dont_decref pto = obj.c_ob_type #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Mon Apr 5 22:56:08 2010 @@ -303,9 +303,9 @@ dealloc = base.c_tp_dealloc # XXX call tp_del if necessary generic_cpy_call(space, dealloc, obj) - if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) + # XXX cpy decrefs the pto here but we do it in the base-dealloc + # hopefully this does not clash with the memory model assumed in + # extension modules @cpython_api([PyObject], lltype.Void, external=False) @@ -344,7 +344,7 @@ obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) pto = rffi.cast(PyObject, type_pto) - Py_DecRef(space, pto) # XXX duplicate decref? (see above) + Py_DecRef(space, pto) def allocate_type_obj(space, w_type): From fijal at codespeak.net Mon Apr 5 23:02:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Apr 2010 23:02:49 +0200 (CEST) Subject: [pypy-svn] r73420 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405210249.98815282B9C@codespeak.net> Author: fijal Date: Mon Apr 5 23:02:48 2010 New Revision: 73420 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: make it false by default, required for translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Mon Apr 5 23:02:48 2010 @@ -18,7 +18,7 @@ class InvalidPointerException(Exception): pass -DEBUG_REFCOUNT = True +DEBUG_REFCOUNT = False def debug_refcount(*args, **kwargs): frame_stackdepth = kwargs.pop("frame_stackdepth", 2) From xoraxax at codespeak.net Mon Apr 5 23:05:59 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 23:05:59 +0200 (CEST) Subject: [pypy-svn] r73421 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100405210559.D5836282B9C@codespeak.net> Author: xoraxax Date: Mon Apr 5 23:05:58 2010 New Revision: 73421 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Remove the skip, real men can live with one failing test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Mon Apr 5 23:05:58 2010 @@ -83,6 +83,9 @@ assert fuu2(u"abc").baz().escape() def test_sre(self): - skip("In Progress") module = self.import_module(name='_sre') + import sre_compile + sre_compile._sre = module + assert sre_compile.MAGIC == module.MAGIC + import re From xoraxax at codespeak.net Mon Apr 5 23:22:28 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 5 Apr 2010 23:22:28 +0200 (CEST) Subject: [pypy-svn] r73422 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100405212228.85E65282B9C@codespeak.net> Author: xoraxax Date: Mon Apr 5 23:22:26 2010 New Revision: 73422 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Log: Comment out decls that are invalid because they are not yet implemented. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Mon Apr 5 23:22:26 2010 @@ -14,10 +14,12 @@ int PyArg_ParseTuple(PyObject *, const char *, ...); int PyArg_VaParse(PyObject *, const char *, va_list); +/* NOT YET IMPLEMENTED int PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, ...); int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, va_list); +*/ #define Py_InitModule(name, methods) \ Py_InitModule4(name, methods, (char *)NULL, (PyObject *)NULL, \ From xoraxax at codespeak.net Tue Apr 6 01:30:39 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:30:39 +0200 (CEST) Subject: [pypy-svn] r73423 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100405233039.C6392282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:30:37 2010 New Revision: 73423 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Add support for PyCObjects. Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h Tue Apr 6 01:30:37 2010 @@ -0,0 +1,5 @@ +typedef struct { + PyObject_HEAD + void (*destructor)(void *); +} PyCObject; + Added: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Tue Apr 6 01:30:37 2010 @@ -0,0 +1,48 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.pyobject import make_ref +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ + cpython_struct, PyObjectFields + + +destructor = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) +PyCObjectStruct = cpython_struct('PyCObject', PyObjectFields + (("destructor", destructor), )) +PyCObject = lltype.Ptr(PyCObjectStruct) + + +class W_PyCObject(Wrappable): + def __init__(self, space): + self.space = space + +class W_PyCObjectFromVoidPtr(W_PyCObject): + def __init__(self, space, voidp): + W_PyCObject.__init__(self, space) + self.voidp = voidp + self.pyo = lltype.nullptr(PyObject.TO) + + def set_pycobject(self, pyo): + self.pyo = pyo + + def __del__(self): + if self.pyo and self.pyo.c_destructor: + self.pyo.c_destructor(self.voidp) + + at cpython_api([rffi.VOIDP_real, destructor], PyObject) +def PyCObject_FromVoidPtr(space, cobj, destr): + """Create a PyCObject from the void * cobj. The destr function + will be called when the object is reclaimed, unless it is NULL.""" + w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj)) + pyo = make_ref(space, w_pycobject) + pycobject = rffi.cast(PyCObject, pyo) + w_pycobject.set_pycobject(pycobject) + pycobject.c_destructor = destr + return pyo + + +W_PyCObject.typedef = TypeDef( + 'PyCObject', + ) +W_PyCObject.typedef.acceptable_as_base_class = False + + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Tue Apr 6 01:30:37 2010 @@ -33,6 +33,7 @@ def make_ref(space, w_obj, borrowed=False, steal=False): from pypy.module.cpyext.typeobject import allocate_type_obj,\ W_PyCTypeObject, PyOLifeline + from pypy.module.cpyext.pycobject import W_PyCObject, PyCObject if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) @@ -81,6 +82,12 @@ py_obj = rffi.cast(PyObject, py_obj_unicode) py_obj.c_ob_refcnt = 1 py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) + elif isinstance(w_obj, W_PyCObject): # a PyCObject + py_cobj = lltype.malloc(PyCObject.TO, flavor='raw', zero=True) + pto = make_ref(space, space.type(w_obj)) + py_obj = rffi.cast(PyObject, py_cobj) + py_obj.c_ob_refcnt = 1 + py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) else: py_obj = lltype.malloc(PyObject.TO, flavor="raw", zero=True) py_obj.c_ob_refcnt = 1 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Tue Apr 6 01:30:37 2010 @@ -0,0 +1,10 @@ +import py + +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.pycobject import destructor + +class TestPyCObject(BaseApiTest): + def test_pycobject(self, space, api): + obj = api.PyCObject_FromVoidPtr(rffi.cast(rffi.VOIDP_real, 0), lltype.nullptr(destructor.TO)) + api.Py_DecRef(obj) From xoraxax at codespeak.net Tue Apr 6 01:31:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:31:29 +0200 (CEST) Subject: [pypy-svn] r73424 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100405233129.B9556282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:31:28 2010 New Revision: 73424 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Import/include the pycobject files and set a few defines and import limits.h. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Tue Apr 6 01:31:28 2010 @@ -52,6 +52,7 @@ import pypy.module.cpyext.iterator import pypy.module.cpyext.unicodeobject import pypy.module.cpyext.misc +import pypy.module.cpyext.pycobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Tue Apr 6 01:31:28 2010 @@ -12,7 +12,6 @@ #define SIZEOF_VOID_P sizeof(void *) #define WITH_DOC_STRINGS #define HAVE_UNICODE -#define INT_MAX (1 << (8 * sizeof(int) - 1)) #define WITHOUT_COMPLEX /* Compat stuff */ @@ -20,6 +19,7 @@ # include # include # include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_DATA(RTYPE) extern RTYPE #else @@ -33,6 +33,7 @@ #define Py_ssize_t long #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) +#define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ @@ -48,6 +49,7 @@ #include +int PyOS_snprintf(char *str, size_t size, const char *format, ...); #include "patchlevel.h" #include "object.h" @@ -76,6 +78,7 @@ #include "unicodeobject.h" #include "eval.h" #include "pymem.h" +#include "pycobject.h" // XXX This shouldn't be included here #include "structmember.h" From xoraxax at codespeak.net Tue Apr 6 01:32:03 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:32:03 +0200 (CEST) Subject: [pypy-svn] r73425 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405233203.36B85282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:32:01 2010 New Revision: 73425 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Log: Half-heartedly implement PyErr_Warn{,Ex}. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 6 01:32:01 2010 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import PyObject, make_ref, register_container from pypy.module.cpyext.state import State from pypy.rlib.rposix import get_errno @@ -104,3 +105,57 @@ violation will occur if no exception has been raised.""" w_type = PyErr_Occurred(space) return PyErr_GivenExceptionMatches(space, w_type, w_exc) + + + at cpython_api([PyObject, rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) +def PyErr_WarnEx(space, w_category, message_ptr, stacklevel): + """Issue a warning message. The category argument is a warning category (see + below) or NULL; the message argument is a message string. stacklevel is a + positive number giving a number of stack frames; the warning will be issued from + the currently executing line of code in that stack frame. A stacklevel of 1 + is the function calling PyErr_WarnEx(), 2 is the function above that, + and so forth. + + This function normally prints a warning message to sys.stderr; however, it is + also possible that the user has specified that warnings are to be turned into + errors, and in that case this will raise an exception. It is also possible that + the function raises an exception because of a problem with the warning machinery + (the implementation imports the warnings module to do the heavy lifting). + The return value is 0 if no exception is raised, or -1 if an exception + is raised. (It is not possible to determine whether a warning message is + actually printed, nor what the reason is for the exception; this is + intentional.) If an exception is raised, the caller should do its normal + exception handling (for example, Py_DECREF() owned references and return + an error value). + + Warning categories must be subclasses of Warning; the default warning + category is RuntimeWarning. The standard Python warning categories are + available as global variables whose names are PyExc_ followed by the Python + exception name. These have the type PyObject*; they are all class + objects. Their names are PyExc_Warning, PyExc_UserWarning, + PyExc_UnicodeWarning, PyExc_DeprecationWarning, + PyExc_SyntaxWarning, PyExc_RuntimeWarning, and + PyExc_FutureWarning. PyExc_Warning is a subclass of + PyExc_Exception; the other warning categories are subclasses of + PyExc_Warning. + + For information about warning control, see the documentation for the + warnings module and the -W option in the command line + documentation. There is no C API for warning control.""" + message = rffi.charp2str(message_ptr) + if category is None: + category = space.gettypeobject(W_RuntimeWarning.typedef) + os.write(2, "WARNING: " + message + "\n") + return 0 + + at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) +def PyErr_Warn(space, w_category, message): + """Issue a warning message. The category argument is a warning category (see + below) or NULL; the message argument is a message string. The warning will + appear to be issued from the function calling PyErr_Warn(), equivalent to + calling PyErr_WarnEx() with a stacklevel of 1. + + Deprecated; use PyErr_WarnEx() instead.""" + return PyErr_WarnEx(w_category, message, 1) + + From xoraxax at codespeak.net Tue Apr 6 01:32:39 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:32:39 +0200 (CEST) Subject: [pypy-svn] r73426 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405233239.13E23282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:32:37 2010 New Revision: 73426 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Log: Implement _Size and stub _Next. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Tue Apr 6 01:32:37 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ + Py_ssize_t, PyObjectP from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -47,3 +48,56 @@ raise OperationError(space.w_KeyError, space.wrap("Key not found")) register_container(space, w_dict) return w_res + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyDict_Size(space, w_obj): + """ + Return the number of items in the dictionary. This is equivalent to + len(p) on a dictionary.""" + return space.int_w(space.len(w_obj)) + + at cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) +def PyDict_Next(space, p, ppos, pkey, pvalue): + """Iterate over all key-value pairs in the dictionary p. The + Py_ssize_t referred to by ppos must be initialized to 0 + prior to the first call to this function to start the iteration; the + function returns true for each pair in the dictionary, and false once all + pairs have been reported. The parameters pkey and pvalue should either + point to PyObject* variables that will be filled in with each key + and value, respectively, or may be NULL. Any references returned through + them are borrowed. ppos should not be altered during iteration. Its + value represents offsets within the internal dictionary structure, and + since the structure is sparse, the offsets are not consecutive. + + For example: + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(self->dict, &pos, &key, &value)) { + /* do something interesting with the values... */ + ... + } + + The dictionary p should not be mutated during iteration. It is safe + (since Python 2.1) to modify the values of the keys as you iterate over the + dictionary, but only so long as the set of keys does not change. For + example: + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(self->dict, &pos, &key, &value)) { + int i = PyInt_AS_LONG(value) + 1; + PyObject *o = PyInt_FromLong(i); + if (o == NULL) + return -1; + if (PyDict_SetItem(self->dict, key, o) < 0) { + Py_DECREF(o); + return -1; + } + Py_DECREF(o); + }""" + raise NotImplementedError + + From xoraxax at codespeak.net Tue Apr 6 01:33:15 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:33:15 +0200 (CEST) Subject: [pypy-svn] r73427 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405233315.A42AE282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:33:14 2010 New Revision: 73427 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Implement PyType_IsSubtype. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 01:33:14 2010 @@ -16,7 +16,7 @@ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - PyUnicodeObject + PyUnicodeObject, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod @@ -33,6 +33,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.rstring import rsplit from pypy.rlib.objectmodel import we_are_translated, specialize +from pypy.module.__builtin__.abstractinst import abstract_issubclass_w WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -492,3 +493,12 @@ __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), __new__ = interp2app(c_type_descr__new__), ) + + at cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) +def PyType_IsSubtype(space, a, b): + """Return true if a is a subtype of b. + """ + w_type1 = from_ref(space, rffi.cast(PyObject, a)) + w_type2 = from_ref(space, rffi.cast(PyObject, b)) + return int(abstract_issubclass_w(space, w_type1, w_type2)) #XXX correct? + From xoraxax at codespeak.net Tue Apr 6 01:34:04 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:34:04 +0200 (CEST) Subject: [pypy-svn] r73428 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405233404.ABED6282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:34:03 2010 New Revision: 73428 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py Log: Implement float checkers. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py Tue Apr 6 01:34:03 2010 @@ -1,7 +1,9 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject +from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers from pypy.interpreter.error import OperationError +PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") + @cpython_api([lltype.Float], PyObject) def PyFloat_FromDouble(space, value): return space.wrap(value) From xoraxax at codespeak.net Tue Apr 6 01:34:24 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:34:24 +0200 (CEST) Subject: [pypy-svn] r73429 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100405233424.590F6282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:34:23 2010 New Revision: 73429 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Log: Implement PySequence_Check PySequence_Size. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Tue Apr 6 01:34:23 2010 @@ -5,6 +5,22 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject + + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PySequence_Check(space, w_obj): + """Return 1 if the object provides sequence protocol, and 0 otherwise. + This function always succeeds.""" + return int(space.findattr(w_obj, space.wrap("__getitem__")) is not None) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PySequence_Size(space, w_obj): + """ + Returns the number of objects in sequence o on success, and -1 on failure. + For objects that do not provide sequence protocol, this is equivalent to the + Python expression len(o).""" + return space.int_w(space.len(w_obj)) + + @cpython_api([PyObject, rffi.CCHARP], PyObject) def PySequence_Fast(space, w_obj, m): """Returns the sequence o as a tuple, unless it is already a tuple or list, in From xoraxax at codespeak.net Tue Apr 6 01:35:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:35:29 +0200 (CEST) Subject: [pypy-svn] r73430 - in pypy/branch/cpython-extension/pypy/module/cpyext: . src test Message-ID: <20100405233529.0B245282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:35:28 2010 New Revision: 73430 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Throw away Maciejs code and copy it from CPython. Also fix the test to expect CPythons behaviour. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Tue Apr 6 01:35:28 2010 @@ -18,9 +18,13 @@ def PyArg_UnpackTuple(): pass + at cpython_api_c() +def PyArg_ParseTupleAndKeywords(): + pass + @cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], rffi.INT_real, error=0) -def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): +def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): #XXX unused, kill me, fijal? arg_i = 0 fmt_i = 0 while True: @@ -46,6 +50,6 @@ elif c == ':': return 1 else: - raise Exception("Unsupported parameter: %s" % (c,)) + raise Exception("Unsupported parameter in vgetargs1: %s" % (c,)) arg_i += 1 fmt_i += 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Tue Apr 6 01:35:28 2010 @@ -19,37 +19,26 @@ int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, va_list); -#ifdef HAVE_DECLSPEC_DLL -/* Export functions */ -/*PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, char *, ...); -PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, char *, ...); -PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, - const char *, char **, ...); -PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); -PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, char *, va_list); -PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, -const char *, char **, va_list);*/ -#endif #define FLAG_COMPAT 1 #define FLAG_SIZE_T 2 /* Forward */ -int pypy_vgetargs1(struct _object *, char *, char *, int); - //static void seterror(int, const char *, int *, const char *, const char *); - //static char *convertitem(PyObject *, const char **, va_list *, int, int *, - // char *, size_t, PyObject **); -//static char *converttuple(PyObject *, const char **, va_list *, int, -// int *, char *, size_t, int, PyObject **); -//static char *convertsimple(PyObject *, const char **, va_list *, int, char *, -// size_t, PyObject **); -//static Py_ssize_t convertbuffer(PyObject *, void **p, char **); -//static int getbuffer(PyObject *, Py_buffer *, char**); - -//static int vgetargskeywords(PyObject *, PyObject *, -// const char *, char **, va_list *, int); -//static char *skipitem(const char **, va_list *, int); +static int vgetargs1(PyObject *, const char *, va_list *, int); +static void seterror(int, const char *, int *, const char *, const char *); +static char *convertitem(PyObject *, const char **, va_list *, int, int *, + char *, size_t, PyObject **); +static char *converttuple(PyObject *, const char **, va_list *, int, + int *, char *, size_t, int, PyObject **); +static char *convertsimple(PyObject *, const char **, va_list *, int, char *, + size_t, PyObject **); +static Py_ssize_t convertbuffer(PyObject *, void **p, char **); +static int getbuffer(PyObject *, Py_buffer *, char**); + +static int vgetargskeywords(PyObject *, PyObject *, + const char *, char **, va_list *, int); +static char *skipitem(const char **, va_list *, int); int PyArg_Parse(PyObject *args, const char *format, ...) @@ -58,7 +47,7 @@ va_list va; va_start(va, format); - retval = pypy_vgetargs1(args, format, (char*)(&va), FLAG_COMPAT); + retval = vgetargs1(args, format, &va, FLAG_COMPAT); va_end(va); return retval; } @@ -70,7 +59,7 @@ va_list va; va_start(va, format); - retval = pypy_vgetargs1(args, format, &va, FLAG_COMPAT|FLAG_SIZE_T); + retval = vgetargs1(args, format, &va, FLAG_COMPAT|FLAG_SIZE_T); va_end(va); return retval; } @@ -83,7 +72,7 @@ va_list va; va_start(va, format); - retval = pypy_vgetargs1(args, format, &va, 0); + retval = vgetargs1(args, format, &va, 0); va_end(va); return retval; } @@ -95,7 +84,7 @@ va_list va; va_start(va, format); - retval = pypy_vgetargs1(args, format, &va, FLAG_SIZE_T); + retval = vgetargs1(args, format, &va, FLAG_SIZE_T); va_end(va); return retval; } @@ -116,7 +105,7 @@ #endif #endif - return pypy_vgetargs1(args, format, &lva, 0); + return vgetargs1(args, format, &lva, 0); } int @@ -134,10 +123,1664 @@ #endif #endif - return pypy_vgetargs1(args, format, &lva, FLAG_SIZE_T); + return vgetargs1(args, format, &lva, FLAG_SIZE_T); +} + + +/* Handle cleanup of allocated memory in case of exception */ + +static void +cleanup_ptr(void *ptr) +{ + PyMem_FREE(ptr); +} + +#if 0 //YYY +static void +cleanup_buffer(void *ptr) +{ + PyBuffer_Release((Py_buffer *) ptr); +} +#endif + +static int +addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *)) +{ + PyObject *cobj; + if (!*freelist) { + *freelist = PyList_New(0); + if (!*freelist) { + destr(ptr); + return -1; + } + } + cobj = PyCObject_FromVoidPtr(ptr, destr); + if (!cobj) { + destr(ptr); + return -1; + } + if (PyList_Append(*freelist, cobj)) { + Py_DECREF(cobj); + return -1; + } + Py_DECREF(cobj); + return 0; +} + +static int +cleanreturn(int retval, PyObject *freelist) +{ + if (freelist && retval != 0) { + /* We were successful, reset the destructors so that they + don't get called. */ + Py_ssize_t len = PyList_GET_SIZE(freelist), i; + for (i = 0; i < len; i++) + ((PyCObject *) PyList_GET_ITEM(freelist, i)) + ->destructor = NULL; + } + Py_XDECREF(freelist); + return retval; +} + + +static int +vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) +{ + char msgbuf[256]; + int levels[32]; + const char *fname = NULL; + const char *message = NULL; + int min = -1; + int max = 0; + int level = 0; + int endfmt = 0; + const char *formatsave = format; + Py_ssize_t i, len; + char *msg; + PyObject *freelist = NULL; + int compat = flags & FLAG_COMPAT; + + assert(compat || (args != (PyObject*)NULL)); + flags = flags & ~FLAG_COMPAT; + + while (endfmt == 0) { + int c = *format++; + switch (c) { + case '(': + if (level == 0) + max++; + level++; + if (level >= 30) + Py_FatalError("too many tuple nesting levels " + "in argument format string"); + break; + case ')': + if (level == 0) + Py_FatalError("excess ')' in getargs format"); + else + level--; + break; + case '\0': + endfmt = 1; + break; + case ':': + fname = format; + endfmt = 1; + break; + case ';': + message = format; + endfmt = 1; + break; + default: + if (level == 0) { + if (c == 'O') + max++; + else if (isalpha(Py_CHARMASK(c))) { + if (c != 'e') /* skip encoded */ + max++; + } else if (c == '|') + min = max; + } + break; + } + } + + if (level != 0) + Py_FatalError(/* '(' */ "missing ')' in getargs format"); + + if (min < 0) + min = max; + + format = formatsave; + + if (compat) { + if (max == 0) { + if (args == NULL) + return 1; + PyOS_snprintf(msgbuf, sizeof(msgbuf), + "%.200s%s takes no arguments", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()"); + PyErr_SetString(PyExc_TypeError, msgbuf); + return 0; + } + else if (min == 1 && max == 1) { + if (args == NULL) { + PyOS_snprintf(msgbuf, sizeof(msgbuf), + "%.200s%s takes at least one argument", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()"); + PyErr_SetString(PyExc_TypeError, msgbuf); + return 0; + } + msg = convertitem(args, &format, p_va, flags, levels, + msgbuf, sizeof(msgbuf), &freelist); + if (msg == NULL) + return cleanreturn(1, freelist); + seterror(levels[0], msg, levels+1, fname, message); + return cleanreturn(0, freelist); + } + else { + PyErr_SetString(PyExc_SystemError, + "old style getargs format uses new features"); + return 0; + } + } + + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_SystemError, + "new style getargs format but argument is not a tuple"); + return 0; + } + + len = PyTuple_GET_SIZE(args); + + if (len < min || max < len) { + if (message == NULL) { + PyOS_snprintf(msgbuf, sizeof(msgbuf), + "%.150s%s takes %s %d argument%s " + "(%ld given)", + fname==NULL ? "function" : fname, + fname==NULL ? "" : "()", + min==max ? "exactly" + : len < min ? "at least" : "at most", + len < min ? min : max, + (len < min ? min : max) == 1 ? "" : "s", + Py_SAFE_DOWNCAST(len, Py_ssize_t, long)); + message = msgbuf; + } + PyErr_SetString(PyExc_TypeError, message); + return 0; + } + + for (i = 0; i < len; i++) { + if (*format == '|') + format++; + msg = convertitem(PyTuple_GET_ITEM(args, i), &format, p_va, + flags, levels, msgbuf, + sizeof(msgbuf), &freelist); + if (msg) { + seterror(i+1, msg, levels, fname, message); + return cleanreturn(0, freelist); + } + } + + if (*format != '\0' && !isalpha(Py_CHARMASK(*format)) && + *format != '(' && + *format != '|' && *format != ':' && *format != ';') { + PyErr_Format(PyExc_SystemError, + "bad format string: %.200s", formatsave); + return cleanreturn(0, freelist); + } + + return cleanreturn(1, freelist); +} + + + +static void +seterror(int iarg, const char *msg, int *levels, const char *fname, + const char *message) +{ + char buf[512]; + int i; + char *p = buf; + + if (PyErr_Occurred()) + return; + else if (message == NULL) { + if (fname != NULL) { + PyOS_snprintf(p, sizeof(buf), "%.200s() ", fname); + p += strlen(p); + } + if (iarg != 0) { + PyOS_snprintf(p, sizeof(buf) - (p - buf), + "argument %d", iarg); + i = 0; + p += strlen(p); + while (levels[i] > 0 && i < 32 && (int)(p-buf) < 220) { + PyOS_snprintf(p, sizeof(buf) - (p - buf), + ", item %d", levels[i]-1); + p += strlen(p); + i++; + } + } + else { + PyOS_snprintf(p, sizeof(buf) - (p - buf), "argument"); + p += strlen(p); + } + PyOS_snprintf(p, sizeof(buf) - (p - buf), " %.256s", msg); + message = buf; + } + PyErr_SetString(PyExc_TypeError, message); +} + + +/* Convert a tuple argument. + On entry, *p_format points to the character _after_ the opening '('. + On successful exit, *p_format points to the closing ')'. + If successful: + *p_format and *p_va are updated, + *levels and *msgbuf are untouched, + and NULL is returned. + If the argument is invalid: + *p_format is unchanged, + *p_va is undefined, + *levels is a 0-terminated list of item numbers, + *msgbuf contains an error message, whose format is: + "must be , not ", where: + is the name of the expected type, and + is the name of the actual type, + and msgbuf is returned. +*/ + +static char * +converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, + int *levels, char *msgbuf, size_t bufsize, int toplevel, + PyObject **freelist) +{ + int level = 0; + int n = 0; + const char *format = *p_format; + int i; + + for (;;) { + int c = *format++; + if (c == '(') { + if (level == 0) + n++; + level++; + } + else if (c == ')') { + if (level == 0) + break; + level--; + } + else if (c == ':' || c == ';' || c == '\0') + break; + else if (level == 0 && isalpha(Py_CHARMASK(c))) + n++; + } + + if (!PySequence_Check(arg) || PyString_Check(arg)) { + levels[0] = 0; + PyOS_snprintf(msgbuf, bufsize, + toplevel ? "expected %d arguments, not %.50s" : + "must be %d-item sequence, not %.50s", + n, + arg == Py_None ? "None" : arg->ob_type->tp_name); + return msgbuf; + } + + if ((i = PySequence_Size(arg)) != n) { + levels[0] = 0; + PyOS_snprintf(msgbuf, bufsize, + toplevel ? "expected %d arguments, not %d" : + "must be sequence of length %d, not %d", + n, i); + return msgbuf; + } + + format = *p_format; + for (i = 0; i < n; i++) { + char *msg; + PyObject *item; + item = PySequence_GetItem(arg, i); + if (item == NULL) { + PyErr_Clear(); + levels[0] = i+1; + levels[1] = 0; + strncpy(msgbuf, "is not retrievable", bufsize); + return msgbuf; + } + msg = convertitem(item, &format, p_va, flags, levels+1, + msgbuf, bufsize, freelist); + /* PySequence_GetItem calls tp->sq_item, which INCREFs */ + Py_XDECREF(item); + if (msg != NULL) { + levels[0] = i+1; + return msg; + } + } + + *p_format = format; + return NULL; +} + + +/* Convert a single item. */ + +static char * +convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, + int *levels, char *msgbuf, size_t bufsize, PyObject **freelist) +{ + char *msg; + const char *format = *p_format; + + if (*format == '(' /* ')' */) { + format++; + msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, + bufsize, 0, freelist); + if (msg == NULL) + format++; + } + else { + msg = convertsimple(arg, &format, p_va, flags, + msgbuf, bufsize, freelist); + if (msg != NULL) + levels[0] = 0; + } + if (msg == NULL) + *p_format = format; + return msg; +} + + + +#define UNICODE_DEFAULT_ENCODING(arg) \ + _PyUnicode_AsDefaultEncodedString(arg, NULL) + +/* Format an error message generated by convertsimple(). */ + +static char * +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +{ + assert(expected != NULL); + assert(arg != NULL); + PyOS_snprintf(msgbuf, bufsize, + "must be %.50s, not %.50s", expected, + arg == Py_None ? "None" : arg->ob_type->tp_name); + return msgbuf; +} + +#define CONV_UNICODE "(unicode conversion error)" + +/* explicitly check for float arguments when integers are expected. For now + * signal a warning. Returns true if an exception was raised. */ +static int +float_argument_error(PyObject *arg) +{ + if (PyFloat_Check(arg) && + PyErr_Warn(PyExc_DeprecationWarning, + "integer argument expected, got float" )) + return 1; + else + return 0; +} + +/* Convert a non-tuple argument. Return NULL if conversion went OK, + or a string with a message describing the failure. The message is + formatted as "must be , not ". + When failing, an exception may or may not have been raised. + Don't call if a tuple is expected. + + When you add new format codes, please don't forget poor skipitem() below. +*/ + +static char * +convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, + char *msgbuf, size_t bufsize, PyObject **freelist) +{ + /* For # codes */ +#define FETCH_SIZE int *q=NULL;Py_ssize_t *q2=NULL;\ + if (flags & FLAG_SIZE_T) q2=va_arg(*p_va, Py_ssize_t*); \ + else q=va_arg(*p_va, int*); +#define STORE_SIZE(s) if (flags & FLAG_SIZE_T) *q2=s; else *q=s; +#define BUFFER_LEN ((flags & FLAG_SIZE_T) ? *q2:*q) + + const char *format = *p_format; + char c = *format++; +#ifdef Py_USING_UNICODE + PyObject *uarg; +#endif + + switch (c) { + +#if 0 + case 'b': { /* unsigned byte -- very short int */ + char *p = va_arg(*p_va, char *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else + *p = (unsigned char) ival; + break; + } + + case 'B': {/* byte sized bitfield - both signed and unsigned + values allowed */ + char *p = va_arg(*p_va, char *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsUnsignedLongMask(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else + *p = (unsigned char) ival; + break; + } + + case 'h': {/* signed short int */ + short *p = va_arg(*p_va, short *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else + *p = (short) ival; + break; + } + + case 'H': { /* short int sized bitfield, both signed and + unsigned allowed */ + unsigned short *p = va_arg(*p_va, unsigned short *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsUnsignedLongMask(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else + *p = (unsigned short) ival; + break; + } +#endif + case 'i': {/* signed int */ + int *p = va_arg(*p_va, int *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else if (ival > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed integer is greater than maximum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else if (ival < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed integer is less than minimum"); + return converterr("integer", arg, msgbuf, bufsize); + } + else + *p = ival; + break; + } +#if 0 + case 'I': { /* int sized bitfield, both signed and + unsigned allowed */ + unsigned int *p = va_arg(*p_va, unsigned int *); + unsigned int ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = (unsigned int)PyInt_AsUnsignedLongMask(arg); + if (ival == (unsigned int)-1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else + *p = ival; + break; + } + + case 'n': /* Py_ssize_t */ +#if SIZEOF_SIZE_T != SIZEOF_LONG + { + Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); + Py_ssize_t ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsSsize_t(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + *p = ival; + break; + } +#endif + /* Fall through from 'n' to 'l' if Py_ssize_t is int */ + case 'l': {/* long int */ + long *p = va_arg(*p_va, long *); + long ival; + if (float_argument_error(arg)) + return converterr("integer", arg, msgbuf, bufsize); + ival = PyInt_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) + return converterr("integer", arg, msgbuf, bufsize); + else + *p = ival; + break; + } + + case 'k': { /* long sized bitfield */ + unsigned long *p = va_arg(*p_va, unsigned long *); + unsigned long ival; + if (PyInt_Check(arg)) + ival = PyInt_AsUnsignedLongMask(arg); + else if (PyLong_Check(arg)) + ival = PyLong_AsUnsignedLongMask(arg); + else + return converterr("integer", arg, msgbuf, bufsize); + *p = ival; + break; + } + +#ifdef HAVE_LONG_LONG + case 'L': {/* PY_LONG_LONG */ + PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); + PY_LONG_LONG ival = PyLong_AsLongLong( arg ); + if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred() ) { + return converterr("long", arg, msgbuf, bufsize); + } else { + *p = ival; + } + break; + } + + case 'K': { /* long long sized bitfield */ + unsigned PY_LONG_LONG *p = va_arg(*p_va, unsigned PY_LONG_LONG *); + unsigned PY_LONG_LONG ival; + if (PyInt_Check(arg)) + ival = PyInt_AsUnsignedLongMask(arg); + else if (PyLong_Check(arg)) + ival = PyLong_AsUnsignedLongLongMask(arg); + else + return converterr("integer", arg, msgbuf, bufsize); + *p = ival; + break; + } +#endif + + case 'f': {/* float */ + float *p = va_arg(*p_va, float *); + double dval = PyFloat_AsDouble(arg); + if (PyErr_Occurred()) + return converterr("float", arg, msgbuf, bufsize); + else + *p = (float) dval; + break; + } + + case 'd': {/* double */ + double *p = va_arg(*p_va, double *); + double dval = PyFloat_AsDouble(arg); + if (PyErr_Occurred()) + return converterr("float", arg, msgbuf, bufsize); + else + *p = dval; + break; + } + +#ifndef WITHOUT_COMPLEX + case 'D': {/* complex double */ + Py_complex *p = va_arg(*p_va, Py_complex *); + Py_complex cval; + cval = PyComplex_AsCComplex(arg); + if (PyErr_Occurred()) + return converterr("complex", arg, msgbuf, bufsize); + else + *p = cval; + break; + } +#endif /* WITHOUT_COMPLEX */ + + case 'c': {/* char */ + char *p = va_arg(*p_va, char *); + if (PyString_Check(arg) && PyString_Size(arg) == 1) + *p = PyString_AS_STRING(arg)[0]; + else + return converterr("char", arg, msgbuf, bufsize); + break; + } + + case 's': {/* string */ + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + if (addcleanup(p, freelist, cleanup_buffer)) { + return converterr( + "(cleanup problem)", + arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { + void **p = (void **)va_arg(*p_va, char **); + FETCH_SIZE; + + if (PyString_Check(arg)) { + *p = PyString_AS_STRING(arg); + STORE_SIZE(PyString_GET_SIZE(arg)); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + *p = PyString_AS_STRING(uarg); + STORE_SIZE(PyString_GET_SIZE(uarg)); + } +#endif + else { /* any buffer-like object */ + char *buf; + Py_ssize_t count = convertbuffer(arg, p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + STORE_SIZE(count); + } + format++; + } else { + char **p = va_arg(*p_va, char **); + + if (PyString_Check(arg)) + *p = PyString_AS_STRING(arg); +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + *p = PyString_AS_STRING(uarg); + } +#endif + else + return converterr("string", arg, msgbuf, bufsize); + if ((Py_ssize_t)strlen(*p) != PyString_Size(arg)) + return converterr("string without null bytes", + arg, msgbuf, bufsize); + } + break; + } + + case 'z': {/* string, may be NULL (None) */ + if (*format == '*') { + Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); + + if (arg == Py_None) + PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); + else if (PyString_Check(arg)) { + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(arg), PyString_GET_SIZE(arg), + 1, 0); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + PyBuffer_FillInfo(p, arg, + PyString_AS_STRING(uarg), PyString_GET_SIZE(uarg), + 1, 0); + } +#endif + else { /* any buffer-like object */ + char *buf; + if (getbuffer(arg, p, &buf) < 0) + return converterr(buf, arg, msgbuf, bufsize); + } + if (addcleanup(p, freelist, cleanup_buffer)) { + return converterr( + "(cleanup problem)", + arg, msgbuf, bufsize); + } + format++; + } else if (*format == '#') { /* any buffer-like object */ + void **p = (void **)va_arg(*p_va, char **); + FETCH_SIZE; + + if (arg == Py_None) { + *p = 0; + STORE_SIZE(0); + } + else if (PyString_Check(arg)) { + *p = PyString_AS_STRING(arg); + STORE_SIZE(PyString_GET_SIZE(arg)); + } +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + *p = PyString_AS_STRING(uarg); + STORE_SIZE(PyString_GET_SIZE(uarg)); + } +#endif + else { /* any buffer-like object */ + char *buf; + Py_ssize_t count = convertbuffer(arg, p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + STORE_SIZE(count); + } + format++; + } else { + char **p = va_arg(*p_va, char **); + + if (arg == Py_None) + *p = 0; + else if (PyString_Check(arg)) + *p = PyString_AS_STRING(arg); +#ifdef Py_USING_UNICODE + else if (PyUnicode_Check(arg)) { + uarg = UNICODE_DEFAULT_ENCODING(arg); + if (uarg == NULL) + return converterr(CONV_UNICODE, + arg, msgbuf, bufsize); + *p = PyString_AS_STRING(uarg); + } +#endif + else + return converterr("string or None", + arg, msgbuf, bufsize); + if (*format == '#') { + FETCH_SIZE; + assert(0); /* XXX redundant with if-case */ + if (arg == Py_None) + *q = 0; + else + *q = PyString_Size(arg); + format++; + } + else if (*p != NULL && + (Py_ssize_t)strlen(*p) != PyString_Size(arg)) + return converterr( + "string without null bytes or None", + arg, msgbuf, bufsize); + } + break; + } + + case 'e': {/* encoded string */ + char **buffer; + const char *encoding; + PyObject *s; + Py_ssize_t size; + int recode_strings; + + /* Get 'e' parameter: the encoding name */ + encoding = (const char *)va_arg(*p_va, const char *); +#ifdef Py_USING_UNICODE + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); +#endif + + /* Get output buffer parameter: + 's' (recode all objects via Unicode) or + 't' (only recode non-string objects) + */ + if (*format == 's') + recode_strings = 1; + else if (*format == 't') + recode_strings = 0; + else + return converterr( + "(unknown parser marker combination)", + arg, msgbuf, bufsize); + buffer = (char **)va_arg(*p_va, char **); + format++; + if (buffer == NULL) + return converterr("(buffer is NULL)", + arg, msgbuf, bufsize); + + /* Encode object */ + if (!recode_strings && PyString_Check(arg)) { + s = arg; + Py_INCREF(s); + } + else { +#ifdef Py_USING_UNICODE + PyObject *u; + + /* Convert object to Unicode */ + u = PyUnicode_FromObject(arg); + if (u == NULL) + return converterr( + "string or unicode or text buffer", + arg, msgbuf, bufsize); + + /* Encode object; use default error handling */ + s = PyUnicode_AsEncodedString(u, + encoding, + NULL); + Py_DECREF(u); + if (s == NULL) + return converterr("(encoding failed)", + arg, msgbuf, bufsize); + if (!PyString_Check(s)) { + Py_DECREF(s); + return converterr( + "(encoder failed to return a string)", + arg, msgbuf, bufsize); + } +#else + return converterr("string", arg, msgbuf, bufsize); +#endif + } + size = PyString_GET_SIZE(s); + + /* Write output; output is guaranteed to be 0-terminated */ + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer of the + needed size is allocated and the data + copied into it; *buffer is updated to point + to the new buffer; the caller is + responsible for PyMem_Free()ing it after + usage + + - if *buffer is not NULL, the data is + copied to *buffer; *buffer_len has to be + set to the size of the buffer on input; + buffer overflow is signalled with an error; + buffer has to provide enough room for the + encoded string plus the trailing 0-byte + + - in both cases, *buffer_len is updated to + the size of the buffer /excluding/ the + trailing 0-byte + + */ + FETCH_SIZE; + + format++; + if (q == NULL && q2 == NULL) { + Py_DECREF(s); + return converterr( + "(buffer_len is NULL)", + arg, msgbuf, bufsize); + } + if (*buffer == NULL) { + *buffer = PyMem_NEW(char, size + 1); + if (*buffer == NULL) { + Py_DECREF(s); + return converterr( + "(memory error)", + arg, msgbuf, bufsize); + } + if (addcleanup(*buffer, freelist, cleanup_ptr)) { + Py_DECREF(s); + return converterr( + "(cleanup problem)", + arg, msgbuf, bufsize); + } + } else { + if (size + 1 > BUFFER_LEN) { + Py_DECREF(s); + return converterr( + "(buffer overflow)", + arg, msgbuf, bufsize); + } + } + memcpy(*buffer, + PyString_AS_STRING(s), + size + 1); + STORE_SIZE(size); + } else { + /* Using a 0-terminated buffer: + + - the encoded string has to be 0-terminated + for this variant to work; if it is not, an + error raised + + - a new buffer of the needed size is + allocated and the data copied into it; + *buffer is updated to point to the new + buffer; the caller is responsible for + PyMem_Free()ing it after usage + + */ + if ((Py_ssize_t)strlen(PyString_AS_STRING(s)) + != size) { + Py_DECREF(s); + return converterr( + "encoded string without NULL bytes", + arg, msgbuf, bufsize); + } + *buffer = PyMem_NEW(char, size + 1); + if (*buffer == NULL) { + Py_DECREF(s); + return converterr("(memory error)", + arg, msgbuf, bufsize); + } + if (addcleanup(*buffer, freelist, cleanup_ptr)) { + Py_DECREF(s); + return converterr("(cleanup problem)", + arg, msgbuf, bufsize); + } + memcpy(*buffer, + PyString_AS_STRING(s), + size + 1); + } + Py_DECREF(s); + break; + } + +#ifdef Py_USING_UNICODE + case 'u': {/* raw unicode buffer (Py_UNICODE *) */ + if (*format == '#') { /* any buffer-like object */ + void **p = (void **)va_arg(*p_va, char **); + FETCH_SIZE; + if (PyUnicode_Check(arg)) { + *p = PyUnicode_AS_UNICODE(arg); + STORE_SIZE(PyUnicode_GET_SIZE(arg)); + } + else { + return converterr("cannot convert raw buffers", + arg, msgbuf, bufsize); + } + format++; + } else { + Py_UNICODE **p = va_arg(*p_va, Py_UNICODE **); + if (PyUnicode_Check(arg)) + *p = PyUnicode_AS_UNICODE(arg); + else + return converterr("unicode", arg, msgbuf, bufsize); + } + break; + } +#endif + + case 'S': { /* string object */ + PyObject **p = va_arg(*p_va, PyObject **); + if (PyString_Check(arg)) + *p = arg; + else + return converterr("string", arg, msgbuf, bufsize); + break; + } + +#ifdef Py_USING_UNICODE + case 'U': { /* Unicode object */ + PyObject **p = va_arg(*p_va, PyObject **); + if (PyUnicode_Check(arg)) + *p = arg; + else + return converterr("unicode", arg, msgbuf, bufsize); + break; + } +#endif +#endif + case 'O': { /* object */ + PyTypeObject *type; + PyObject **p; + if (*format == '!') { + type = va_arg(*p_va, PyTypeObject*); + p = va_arg(*p_va, PyObject **); + format++; + if (PyType_IsSubtype(arg->ob_type, type)) + *p = arg; + else + return converterr(type->tp_name, arg, msgbuf, bufsize); + + } + else if (*format == '?') { + inquiry pred = va_arg(*p_va, inquiry); + p = va_arg(*p_va, PyObject **); + format++; + if ((*pred)(arg)) + *p = arg; + else + return converterr("(unspecified)", + arg, msgbuf, bufsize); + + } + else if (*format == '&') { + typedef int (*converter)(PyObject *, void *); + converter convert = va_arg(*p_va, converter); + void *addr = va_arg(*p_va, void *); + format++; + if (! (*convert)(arg, addr)) + return converterr("(unspecified)", + arg, msgbuf, bufsize); + } + else { + p = va_arg(*p_va, PyObject **); + *p = arg; + } + break; + } + +#if 0 + case 'w': { /* memory buffer, read-write access */ + void **p = va_arg(*p_va, void **); + void *res; + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + Py_ssize_t count; + + if (pb && pb->bf_releasebuffer && *format != '*') + /* Buffer must be released, yet caller does not use + the Py_buffer protocol. */ + return converterr("pinned buffer", arg, msgbuf, bufsize); + + if (pb && pb->bf_getbuffer && *format == '*') { + /* Caller is interested in Py_buffer, and the object + supports it directly. */ + format++; + if (pb->bf_getbuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { + PyErr_Clear(); + return converterr("read-write buffer", arg, msgbuf, bufsize); + } + if (addcleanup(p, freelist, cleanup_buffer)) { + return converterr( + "(cleanup problem)", + arg, msgbuf, bufsize); + } + if (!PyBuffer_IsContiguous((Py_buffer*)p, 'C')) + return converterr("contiguous buffer", arg, msgbuf, bufsize); + break; + } + + if (pb == NULL || + pb->bf_getwritebuffer == NULL || + pb->bf_getsegcount == NULL) + return converterr("read-write buffer", arg, msgbuf, bufsize); + if ((*pb->bf_getsegcount)(arg, NULL) != 1) + return converterr("single-segment read-write buffer", + arg, msgbuf, bufsize); + if ((count = pb->bf_getwritebuffer(arg, 0, &res)) < 0) + return converterr("(unspecified)", arg, msgbuf, bufsize); + if (*format == '*') { + PyBuffer_FillInfo((Py_buffer*)p, arg, res, count, 1, 0); + format++; + } + else { + *p = res; + if (*format == '#') { + FETCH_SIZE; + STORE_SIZE(count); + format++; + } + } + break; + } + + case 't': { /* 8-bit character buffer, read-only access */ + char **p = va_arg(*p_va, char **); + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + Py_ssize_t count; + + if (*format++ != '#') + return converterr( + "invalid use of 't' format character", + arg, msgbuf, bufsize); + if (!PyType_HasFeature(arg->ob_type, + Py_TPFLAGS_HAVE_GETCHARBUFFER) || + pb == NULL || pb->bf_getcharbuffer == NULL || + pb->bf_getsegcount == NULL) + return converterr( + "string or read-only character buffer", + arg, msgbuf, bufsize); + + if (pb->bf_getsegcount(arg, NULL) != 1) + return converterr( + "string or single-segment read-only buffer", + arg, msgbuf, bufsize); + + if (pb->bf_releasebuffer) + return converterr( + "string or pinned buffer", + arg, msgbuf, bufsize); + + count = pb->bf_getcharbuffer(arg, 0, p); + if (count < 0) + return converterr("(unspecified)", arg, msgbuf, bufsize); + { + FETCH_SIZE; + STORE_SIZE(count); + } + break; + } +#endif + default: + return converterr("impossible", arg, msgbuf, bufsize); + + } + + *p_format = format; + return NULL; +} + +static Py_ssize_t +convertbuffer(PyObject *arg, void **p, char **errmsg) +{ + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + Py_ssize_t count; + if (pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL || + pb->bf_releasebuffer != NULL) { + *errmsg = "string or read-only buffer"; + return -1; + } + if ((*pb->bf_getsegcount)(arg, NULL) != 1) { + *errmsg = "string or single-segment read-only buffer"; + return -1; + } + if ((count = (*pb->bf_getreadbuffer)(arg, 0, p)) < 0) { + *errmsg = "(unspecified)"; + } + return count; +} + +#if 0 //YYY +static int +getbuffer(PyObject *arg, Py_buffer *view, char **errmsg) +{ + void *buf; + Py_ssize_t count; + PyBufferProcs *pb = arg->ob_type->tp_as_buffer; + if (pb == NULL) { + *errmsg = "string or buffer"; + return -1; + } + if (pb->bf_getbuffer) { + if (pb->bf_getbuffer(arg, view, 0) < 0) { + *errmsg = "convertible to a buffer"; + return -1; + } + if (!PyBuffer_IsContiguous(view, 'C')) { + *errmsg = "contiguous buffer"; + return -1; + } + return 0; + } + + count = convertbuffer(arg, &buf, errmsg); + if (count < 0) { + *errmsg = "convertible to a buffer"; + return count; + } + PyBuffer_FillInfo(view, NULL, buf, count, 1, 0); + return 0; +} +#endif + +/* Support for keyword arguments donated by + Geoff Philbrick */ + +/* Return false (0) for error, else true. */ +int +PyArg_ParseTupleAndKeywords(PyObject *args, + PyObject *keywords, + const char *format, + char **kwlist, ...) +{ + int retval; + va_list va; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + format == NULL || + kwlist == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, kwlist); + retval = vgetargskeywords(args, keywords, format, kwlist, &va, 0); + va_end(va); + return retval; +} + +int +_PyArg_ParseTupleAndKeywords_SizeT(PyObject *args, + PyObject *keywords, + const char *format, + char **kwlist, ...) +{ + int retval; + va_list va; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + format == NULL || + kwlist == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, kwlist); + retval = vgetargskeywords(args, keywords, format, + kwlist, &va, FLAG_SIZE_T); + va_end(va); + return retval; +} + + +int +PyArg_VaParseTupleAndKeywords(PyObject *args, + PyObject *keywords, + const char *format, + char **kwlist, va_list va) +{ + int retval; + va_list lva; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + format == NULL || + kwlist == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + +#ifdef VA_LIST_IS_ARRAY + memcpy(lva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(lva, va); +#else + lva = va; +#endif +#endif + + retval = vgetargskeywords(args, keywords, format, kwlist, &lva, 0); + return retval; } int +_PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args, + PyObject *keywords, + const char *format, + char **kwlist, va_list va) +{ + int retval; + va_list lva; + + if ((args == NULL || !PyTuple_Check(args)) || + (keywords != NULL && !PyDict_Check(keywords)) || + format == NULL || + kwlist == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + +#ifdef VA_LIST_IS_ARRAY + memcpy(lva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(lva, va); +#else + lva = va; +#endif +#endif + + retval = vgetargskeywords(args, keywords, format, + kwlist, &lva, FLAG_SIZE_T); + return retval; +} + +#define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') + +static int +vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, + char **kwlist, va_list *p_va, int flags) +{ + char msgbuf[512]; + int levels[32]; + const char *fname, *msg, *custom_msg, *keyword; + int min = INT_MAX; + int i, len, nargs, nkeywords; + PyObject *freelist = NULL, *current_arg; + + assert(args != NULL && PyTuple_Check(args)); + assert(keywords == NULL || PyDict_Check(keywords)); + assert(format != NULL); + assert(kwlist != NULL); + assert(p_va != NULL); + + /* grab the function name or custom error msg first (mutually exclusive) */ + fname = strchr(format, ':'); + if (fname) { + fname++; + custom_msg = NULL; + } + else { + custom_msg = strchr(format,';'); + if (custom_msg) + custom_msg++; + } + + /* scan kwlist and get greatest possible nbr of args */ + for (len=0; kwlist[len]; len++) + continue; + + nargs = PyTuple_GET_SIZE(args); + nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords); + if (nargs + nkeywords > len) { + PyErr_Format(PyExc_TypeError, "%s%s takes at most %d " + "argument%s (%d given)", + (fname == NULL) ? "function" : fname, + (fname == NULL) ? "" : "()", + len, + (len == 1) ? "" : "s", + nargs + nkeywords); + return 0; + } + + /* convert tuple args and keyword args in same loop, using kwlist to drive process */ + for (i = 0; i < len; i++) { + keyword = kwlist[i]; + if (*format == '|') { + min = i; + format++; + } + if (IS_END_OF_FORMAT(*format)) { + PyErr_Format(PyExc_RuntimeError, + "More keyword list entries (%d) than " + "format specifiers (%d)", len, i); + return cleanreturn(0, freelist); + } + current_arg = NULL; + if (nkeywords) { + current_arg = PyDict_GetItemString(keywords, keyword); + } + if (current_arg) { + --nkeywords; + if (i < nargs) { + /* arg present in tuple and in dict */ + PyErr_Format(PyExc_TypeError, + "Argument given by name ('%s') " + "and position (%d)", + keyword, i+1); + return cleanreturn(0, freelist); + } + } + else if (nkeywords && PyErr_Occurred()) + return cleanreturn(0, freelist); + else if (i < nargs) + current_arg = PyTuple_GET_ITEM(args, i); + + if (current_arg) { + msg = convertitem(current_arg, &format, p_va, flags, + levels, msgbuf, sizeof(msgbuf), &freelist); + if (msg) { + seterror(i+1, msg, levels, fname, custom_msg); + return cleanreturn(0, freelist); + } + continue; + } + + if (i < min) { + PyErr_Format(PyExc_TypeError, "Required argument " + "'%s' (pos %d) not found", + keyword, i+1); + return cleanreturn(0, freelist); + } + /* current code reports success when all required args + * fulfilled and no keyword args left, with no further + * validation. XXX Maybe skip this in debug build ? + */ + if (!nkeywords) + return cleanreturn(1, freelist); + + /* We are into optional args, skip thru to any remaining + * keyword args */ + msg = skipitem(&format, p_va, flags); + if (msg) { + PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg, + format); + return cleanreturn(0, freelist); + } + } + + if (!IS_END_OF_FORMAT(*format) && *format != '|') { + PyErr_Format(PyExc_RuntimeError, + "more argument specifiers than keyword list entries " + "(remaining format:'%s')", format); + return cleanreturn(0, freelist); + } + + /* make sure there are no extraneous keyword arguments */ + if (nkeywords > 0) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(keywords, &pos, &key, &value)) { + int match = 0; + char *ks; + if (!PyString_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, freelist); + } + ks = PyString_AsString(key); + for (i = 0; i < len; i++) { + if (!strcmp(ks, kwlist[i])) { + match = 1; + break; + } + } + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%s' is an invalid keyword " + "argument for this function", + ks); + return cleanreturn(0, freelist); + } + } + } + + return cleanreturn(1, freelist); +} + + +static char * +skipitem(const char **p_format, va_list *p_va, int flags) +{ + const char *format = *p_format; + char c = *format++; + + switch (c) { + + /* simple codes + * The individual types (second arg of va_arg) are irrelevant */ + + case 'b': /* byte -- very short int */ + case 'B': /* byte as bitfield */ + case 'h': /* short int */ + case 'H': /* short int as bitfield */ + case 'i': /* int */ + case 'I': /* int sized bitfield */ + case 'l': /* long int */ + case 'k': /* long int sized bitfield */ +#ifdef HAVE_LONG_LONG + case 'L': /* PY_LONG_LONG */ + case 'K': /* PY_LONG_LONG sized bitfield */ +#endif + case 'f': /* float */ + case 'd': /* double */ +#ifndef WITHOUT_COMPLEX + case 'D': /* complex double */ +#endif + case 'c': /* char */ + { + (void) va_arg(*p_va, void *); + break; + } + + case 'n': /* Py_ssize_t */ + { + (void) va_arg(*p_va, Py_ssize_t *); + break; + } + + /* string codes */ + + case 'e': /* string with encoding */ + { + (void) va_arg(*p_va, const char *); + if (!(*format == 's' || *format == 't')) + /* after 'e', only 's' and 't' is allowed */ + goto err; + format++; + /* explicit fallthrough to string cases */ + } + + case 's': /* string */ + case 'z': /* string or None */ +#ifdef Py_USING_UNICODE + case 'u': /* unicode string */ +#endif + case 't': /* buffer, read-only */ + case 'w': /* buffer, read-write */ + { + (void) va_arg(*p_va, char **); + if (*format == '#') { + if (flags & FLAG_SIZE_T) + (void) va_arg(*p_va, Py_ssize_t *); + else + (void) va_arg(*p_va, int *); + format++; + } else if ((c == 's' || c == 'z') && *format == '*') { + format++; + } + break; + } + + /* object codes */ + + case 'S': /* string object */ +#ifdef Py_USING_UNICODE + case 'U': /* unicode string object */ +#endif + { + (void) va_arg(*p_va, PyObject **); + break; + } + + case 'O': /* object */ + { + if (*format == '!') { + format++; + (void) va_arg(*p_va, PyTypeObject*); + (void) va_arg(*p_va, PyObject **); + } +#if 0 +/* I don't know what this is for */ + else if (*format == '?') { + inquiry pred = va_arg(*p_va, inquiry); + format++; + if ((*pred)(arg)) { + (void) va_arg(*p_va, PyObject **); + } + } +#endif + else if (*format == '&') { + typedef int (*converter)(PyObject *, void *); + (void) va_arg(*p_va, converter); + (void) va_arg(*p_va, void *); + format++; + } + else { + (void) va_arg(*p_va, PyObject **); + } + break; + } + + case '(': /* bypass tuple, not handled at all previously */ + { + char *msg; + for (;;) { + if (*format==')') + break; + if (IS_END_OF_FORMAT(*format)) + return "Unmatched left paren in format " + "string"; + msg = skipitem(&format, p_va, flags); + if (msg) + return msg; + } + format++; + break; + } + + case ')': + return "Unmatched right paren in format string"; + + default: +err: + return "impossible"; + + } + + *p_format = format; + return NULL; +} + + +int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) { Py_ssize_t i, l; @@ -194,6 +1837,30 @@ } va_end(vargs); return 1; +} + +/* For type constructors that don't take keyword args + * + * Sets a TypeError and returns 0 if the kwds dict is + * not empty, returns 1 otherwise + */ +int +_PyArg_NoKeywords(const char *funcname, PyObject *kw) +{ + if (kw == NULL) + return 1; + if (!PyDict_CheckExact(kw)) { + PyErr_BadInternalCall(); + return 0; + } + if (PyDict_Size(kw) == 0) + return 1; + + PyErr_Format(PyExc_TypeError, "%s does not take keyword arguments", + funcname); + return 0; } -// REST IS NOT COPIED FROM CPYTHON +#ifdef __cplusplus +}; +#endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Tue Apr 6 01:35:28 2010 @@ -8,7 +8,7 @@ ('oneargint', 'METH_VARARGS', ''' int l; - if (!PyArg_Parse(args, "i", &l)) { + if (!PyArg_ParseTuple(args, "i", &l)) { return NULL; } return PyInt_FromLong(l); @@ -25,7 +25,7 @@ ('oneargobject', 'METH_VARARGS', ''' PyObject *obj; - if (!PyArg_Parse(args, "O", &obj)) { + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } Py_INCREF(obj); @@ -34,7 +34,7 @@ ('oneargobjectandlisttype', 'METH_VARARGS', ''' PyObject *obj; - if (!PyArg_Parse(args, "O!", &PyList_Type, &obj)) { + if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &obj)) { return NULL; } Py_INCREF(obj); @@ -42,12 +42,7 @@ ''')]) assert mod.oneargint(1) == 1 raises(TypeError, mod.oneargint, None) - try: - mod.oneargint() - except IndexError: - pass - else: - raise Exception("DID NOT RAISE") + raises(TypeError, mod.oneargint) assert mod.oneargandform(1) == 1 sentinel = object() From xoraxax at codespeak.net Tue Apr 6 01:37:58 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:37:58 +0200 (CEST) Subject: [pypy-svn] r73431 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100405233758.9BA06282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:37:57 2010 New Revision: 73431 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Log: Enable the new functions in modsupport.h (WTF, weird choice to put them there). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Tue Apr 6 01:37:57 2010 @@ -14,12 +14,10 @@ int PyArg_ParseTuple(PyObject *, const char *, ...); int PyArg_VaParse(PyObject *, const char *, va_list); -/* NOT YET IMPLEMENTED int PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, ...); int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, va_list); -*/ #define Py_InitModule(name, methods) \ Py_InitModule4(name, methods, (char *)NULL, (PyObject *)NULL, \ From xoraxax at codespeak.net Tue Apr 6 01:42:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 01:42:05 +0200 (CEST) Subject: [pypy-svn] r73432 - in pypy/branch/cpython-extension/pypy/module/cpyext: include src Message-ID: <20100405234205.91071282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 01:42:04 2010 New Revision: 73432 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Log: Enable new type monikers, define a few sizes. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Tue Apr 6 01:42:04 2010 @@ -34,6 +34,8 @@ #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) +#define SIZEOF_SIZE_T 4 +#define SIZEOF_LONG 4 /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Tue Apr 6 01:42:04 2010 @@ -667,7 +667,7 @@ *p = ival; break; } - +#endif case 'n': /* Py_ssize_t */ #if SIZEOF_SIZE_T != SIZEOF_LONG { @@ -696,6 +696,7 @@ break; } +#if 0 case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); unsigned long ival; From xoraxax at codespeak.net Tue Apr 6 02:09:03 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 02:09:03 +0200 (CEST) Subject: [pypy-svn] r73433 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100406000903.21ACE282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 02:09:01 2010 New Revision: 73433 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Implement missing function in object.py, disable the corresponding macros. Add a hack to accept also un-readied typeobjects. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Tue Apr 6 02:09:01 2010 @@ -410,15 +410,20 @@ #define PyObject_INIT_VAR(op, typeobj, size) \ ( Py_SIZE(op) = (size), PyObject_INIT((op), (typeobj)) ) +/* #define PyObject_NEW(type, typeobj) \ ( (type *) PyObject_Init( \ (PyObject *) PyObject_MALLOC( _PyObject_SIZE(typeobj) ), (typeobj)) ) +*/ +#define PyObject_NEW PyObject_New +#define PyObject_NEW_VAR PyObject_NewVar +/* #define PyObject_NEW_VAR(type, typeobj, n) \ ( (type *) PyObject_InitVar( \ (PyVarObject *) PyObject_MALLOC(_PyObject_VAR_SIZE((typeobj),(n)) ),\ (typeobj), (n)) ) - +*/ /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Tue Apr 6 02:09:01 2010 @@ -11,12 +11,20 @@ import pypy.module.__builtin__.operation as operation - at cpython_api([PyObject], PyObject) -def _PyObject_New(space, w_type): + at cpython_api([PyTypeObjectPtr], PyObject) +def _PyObject_New(space, type): + try: + w_type = from_ref(space, rffi.cast(PyObject, type)) + except: + import pdb; pdb.set_trace() if isinstance(w_type, W_PyCTypeObject): w_obj = space.allocate_instance(W_ObjectObject, w_type) return w_obj - assert False, "Please add more cases in get_cls_for_type_object!" + assert False, "Please add more cases in _PyObject_New" + + at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) +def _PyObject_NewVar(space, type, size): # XXX use size! + return _PyObject_New(space, type) @cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_Del(space, obj): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Tue Apr 6 02:09:01 2010 @@ -123,6 +123,7 @@ def from_ref(space, ref): + from pypy.module.cpyext.typeobject import PyPyType_Ready assert lltype.typeOf(ref) == PyObject if not ref: return None @@ -132,13 +133,20 @@ w_obj = state.py_objects_r2w[ptr] except KeyError: ref_type = rffi.cast(PyObject, ref.c_ob_type) - if ref != ref_type and space.is_w(from_ref(space, ref_type), space.w_str): - return force_string(space, ref) + if ref != ref_type: + w_type = from_ref(space, ref_type) + if space.is_w(w_type, space.w_str): + return force_string(space, ref) + elif space.is_w(w_type, space.w_type): + PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) + return from_ref(space, ref) + else: + msg = "" + if not we_are_translated(): + msg = "Got invalid reference to a PyObject: %r" % (ref, ) + raise InvalidPointerException(msg) else: - msg = "" - if not we_are_translated(): - msg = "Got invalid reference to a PyObject: %r" % (ref, ) - raise InvalidPointerException(msg) + raise InvalidPointerException("This should never happen") return w_obj From xoraxax at codespeak.net Tue Apr 6 03:13:14 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:13:14 +0200 (CEST) Subject: [pypy-svn] r73434 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406011314.2327A282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:13:12 2010 New Revision: 73434 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Also allow llvalues as return values in the wrapper. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 03:13:12 2010 @@ -360,7 +360,8 @@ if callable.api_func.restype is PyObject: borrowed = callable.api_func.borrowed - retval = make_ref(space, retval, borrowed=borrowed) + if not rffi._isllptr(retval): + retval = make_ref(space, retval, borrowed=borrowed) if borrowed: try: add_borrowed_object(space, retval) From xoraxax at codespeak.net Tue Apr 6 03:13:34 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:13:34 +0200 (CEST) Subject: [pypy-svn] r73435 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406011334.DE7FC282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:13:33 2010 New Revision: 73435 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Rework allocation in object.py. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Tue Apr 6 03:13:33 2010 @@ -13,19 +13,16 @@ @cpython_api([PyTypeObjectPtr], PyObject) def _PyObject_New(space, type): - try: - w_type = from_ref(space, rffi.cast(PyObject, type)) - except: - import pdb; pdb.set_trace() + return _PyObject_NewVar(space, type, 0) + + at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) +def _PyObject_NewVar(space, type, size): + w_type = from_ref(space, rffi.cast(PyObject, type)) if isinstance(w_type, W_PyCTypeObject): w_obj = space.allocate_instance(W_ObjectObject, w_type) - return w_obj + return make_ref(space, w_obj, items=size) assert False, "Please add more cases in _PyObject_New" - at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) -def _PyObject_NewVar(space, type, size): # XXX use size! - return _PyObject_New(space, type) - @cpython_api([rffi.VOIDP_real], lltype.Void) def PyObject_Del(space, obj): lltype.free(obj, flavor='raw') From xoraxax at codespeak.net Tue Apr 6 03:14:02 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:14:02 +0200 (CEST) Subject: [pypy-svn] r73436 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406011402.135DD282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:14:00 2010 New Revision: 73436 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: Special case __getattr__ and create a wrapper for it. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Tue Apr 6 03:14:00 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import generic_cpy_call, cpython_api, \ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ - ternaryfunc, PyTypeObjectPtr + ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt @@ -29,6 +29,22 @@ check_num_args(space, w_args, 0) return generic_cpy_call(space, func_unary, w_self) +def wrap_binaryfunc(space, w_self, w_args, func): + func_binary = rffi.cast(binaryfunc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + return generic_cpy_call(space, func_binary, w_self, args_w[0]) + +def wrap_getattr(space, w_self, w_args, func): + func_target = rffi.cast(getattrfunc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + name_ptr = rffi.str2charp(space.str_w(args_w[0])) + try: + return generic_cpy_call(space, func_target, w_self, name_ptr) + finally: + lltype.free(name_ptr, flavor="raw") + def wrap_call(space, w_self, w_args, func, w_kwds): func_target = rffi.cast(ternaryfunc, func) return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) @@ -48,14 +64,24 @@ # adopted from typeobject.c def FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, FLAGS): - wrapper = globals().get(WRAPPER, None) + wrapper = globals().get(WRAPPER, Ellipsis) + if WRAPPER is None: + wrapper = None + if NAME == "__getattr__": + wrapper = wrap_getattr function = globals().get(FUNCTION, None) slotname = ("c_" + SLOT).split(".") assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS if FLAGS: + if wrapper is Ellipsis: + def wrapper(space, w_self, w_args, func, w_kwds): + raise NotImplementedError("Wrapper for slot " + NAME) wrapper1 = None wrapper2 = wrapper else: + if wrapper is Ellipsis: + def wrapper(space, w_self, w_args, func): + raise NotImplementedError("Wrapper for slot " + NAME) wrapper1 = wrapper wrapper2 = None return (NAME, slotname, function, wrapper1, wrapper2, DOC) From xoraxax at codespeak.net Tue Apr 6 03:14:21 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:14:21 +0200 (CEST) Subject: [pypy-svn] r73437 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406011421.37B0C282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:14:19 2010 New Revision: 73437 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Correctly calculate allocation size. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Tue Apr 6 03:14:19 2010 @@ -30,7 +30,7 @@ print >>sys.stderr -def make_ref(space, w_obj, borrowed=False, steal=False): +def make_ref(space, w_obj, borrowed=False, steal=False, items=0): from pypy.module.cpyext.typeobject import allocate_type_obj,\ W_PyCTypeObject, PyOLifeline from pypy.module.cpyext.pycobject import W_PyCObject, PyCObject @@ -59,7 +59,7 @@ # Don't increase refcount for non-heaptypes if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: Py_DecRef(space, w_type_pyo) - basicsize = pto.c_tp_basicsize + basicsize = pto.c_tp_basicsize + items * pto.c_tp_itemsize py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, flavor="raw", zero=True) py_obj = rffi.cast(PyObject, py_obj_pad) From xoraxax at codespeak.net Tue Apr 6 03:15:23 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:15:23 +0200 (CEST) Subject: [pypy-svn] r73438 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406011523.0AEE0282B90@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:15:21 2010 New Revision: 73438 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Remove error message. The wrappers contain already an exception raiser if they are not implemented. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 03:15:21 2010 @@ -148,7 +148,6 @@ if not func: continue if wrapper_func is None and wrapper_func_kwds is None: - os.write(2, method_name + " used by the type but no wrapper function defined!\n") continue dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, func_voidp) From xoraxax at codespeak.net Tue Apr 6 03:15:45 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 03:15:45 +0200 (CEST) Subject: [pypy-svn] r73439 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406011545.2DFDE282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 03:15:43 2010 New Revision: 73439 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix test failures by clearing non_heaptypes. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Tue Apr 6 03:15:43 2010 @@ -195,6 +195,7 @@ state = self.space.fromcache(State) for w_obj in state.non_heaptypes: Py_DecRef(self.space, w_obj) + state.non_heaptypes[:] = [] except OperationError: pass except AttributeError: From fijal at codespeak.net Tue Apr 6 05:28:38 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 05:28:38 +0200 (CEST) Subject: [pypy-svn] r73440 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406032838.D4AE6282B90@codespeak.net> Author: fijal Date: Tue Apr 6 05:28:36 2010 New Revision: 73440 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Log: Improve the test. It's copied from cpython so should work, but since I already wrote it we don't loose anything by having it. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_getargs.py Tue Apr 6 05:28:36 2010 @@ -39,6 +39,18 @@ } Py_INCREF(obj); return obj; + '''), + ('twoopt', 'METH_VARARGS', + ''' + PyObject *a; + PyObject *b = NULL; + if (!PyArg_ParseTuple(args, "O|O", &a, &b)) { + return NULL; + } + if (!b) { + b = PyInt_FromLong(42); + } + return b; ''')]) assert mod.oneargint(1) == 1 raises(TypeError, mod.oneargint, None) @@ -49,3 +61,6 @@ res = mod.oneargobject(sentinel) raises(TypeError, "mod.oneargobjectandlisttype(sentinel)") assert res is sentinel + assert mod.twoopt(1) == 42 + assert mod.twoopt(1, 2) == 2 + raises(TypeError, mod.twoopt, 1, 2, 3) From fijal at codespeak.net Tue Apr 6 05:40:02 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 05:40:02 +0200 (CEST) Subject: [pypy-svn] r73441 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406034002.D5062282B90@codespeak.net> Author: fijal Date: Tue Apr 6 05:40:00 2010 New Revision: 73441 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/misc.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Log: Some cleanups and refactorings - kill cpython_api_c and replace it with register_c_function. I think now it's sort of pointless to have some files Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 05:40:00 2010 @@ -207,13 +207,11 @@ return unwrapper_raise # used in 'normal' RPython code. return decorate -def cpython_api_c(): - def decorate(func): - def uncallable(*args, **kwds): - raise Exception("Uncallable") - FUNCTIONS_C[func.func_name] = None - return uncallable - return decorate +def register_c_function(name): + """ Register a function implemented in C as a part of API + (not yet callable from RPython) + """ + FUNCTIONS_C[name] = None def cpython_struct(name, fields, forward=None): configname = name.replace(' ', '__') Modified: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py Tue Apr 6 05:40:00 2010 @@ -1,55 +1,11 @@ from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL, \ - VA_LIST_P, cpython_api_c + VA_LIST_P, register_c_function from pypy.module.cpyext import api from pypy.module.cpyext.pyobject import from_ref, make_ref,\ add_borrowed_object, register_container from pypy.rpython.lltypesystem import lltype, rffi - at cpython_api_c() -def PyArg_Parse(): - pass - - at cpython_api_c() -def PyArg_ParseTuple(): - pass - - at cpython_api_c() -def PyArg_UnpackTuple(): - pass - - at cpython_api_c() -def PyArg_ParseTupleAndKeywords(): - pass - - at cpython_api([PyObject, rffi.CCHARP, VA_LIST_P, rffi.INT_real], - rffi.INT_real, error=0) -def pypy_vgetargs1(space, w_obj, fmt, va_list_p, flags): #XXX unused, kill me, fijal? - arg_i = 0 - fmt_i = 0 - while True: - c = fmt[fmt_i] - if c == "\x00": - return 1 - elif c == "i": - arr = api.va_get_int_star(va_list_p) - arr[0] = rffi.cast(rffi.INT, - space.int_w(space.getitem(w_obj, space.wrap(arg_i)))) - elif c == "O": - w_item = space.getitem(w_obj, space.wrap(arg_i)) - if fmt[fmt_i + 1] == "!": - fmt_i += 1 - w_type = from_ref(space, api.va_get_PyObject_star(va_list_p)) - if not space.is_true(space.isinstance(w_item, w_type)): - raise OperationError(space.w_TypeError, - space.wrap("wrong type")) - arr = api.va_get_PyObject_star_star(va_list_p) - arr[0] = make_ref(space, w_item, borrowed=True) - register_container(space, w_obj) - add_borrowed_object(space, arr[0]) - elif c == ':': - return 1 - else: - raise Exception("Unsupported parameter in vgetargs1: %s" % (c,)) - arg_i += 1 - fmt_i += 1 +for name in ['PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', + 'PyArg_ParseTupleAndKeywords']: + register_c_function(name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/misc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/misc.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/misc.py Tue Apr 6 05:40:00 2010 @@ -1,14 +1,5 @@ -from pypy.module.cpyext.api import cpython_api_c +from pypy.module.cpyext.api import register_c_function - at cpython_api_c() -def Py_FatalError(): - pass - - at cpython_api_c() -def PyOS_snprintf(): - pass - - at cpython_api_c() -def PyOS_vsnprintf(): - pass +for name in ['Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf']: + register_c_function(name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Tue Apr 6 05:40:00 2010 @@ -1,19 +1,14 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cpython_api_c + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, register_c_function from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError - at cpython_api_c() -def PyModule_AddObject(): - pass - - at cpython_api_c() -def Py_BuildValue(): - pass +for name in ['PyModule_AddObject', 'Py_BuildValue']: + register_c_function(name) def PyImport_AddModule(space, name): w_name = space.wrap(name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Tue Apr 6 05:40:00 2010 @@ -3,7 +3,8 @@ from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, build_type_checkers, - PyObjectP, cpython_api_c, PyTypeObjectPtr) + PyObjectP, PyTypeObjectPtr, + register_c_function) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef @@ -20,9 +21,7 @@ py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str - at cpython_api_c() -def PyString_FromFormatV(): - pass +register_c_function('PyString_FromFormatV') @cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 05:40:00 2010 @@ -12,7 +12,7 @@ from pypy.objspace.std.typetype import _precheck_for_new from pypy.objspace.std.objectobject import W_ObjectObject from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ +from pypy.module.cpyext.api import cpython_api, cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Tue Apr 6 05:40:00 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.lltypesystem.lltype import Ptr, FuncType, Void -from pypy.module.cpyext.api import cpython_api, cpython_api_c, cpython_struct, \ +from pypy.module.cpyext.api import cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ PyTypeObject, PyTypeObjectPtr From fijal at codespeak.net Tue Apr 6 05:55:42 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 05:55:42 +0200 (CEST) Subject: [pypy-svn] r73442 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406035542.57A77282B90@codespeak.net> Author: fijal Date: Tue Apr 6 05:55:40 2010 New Revision: 73442 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: disable the va_get_xxx since nobody uses them any more. They're however a feature that we might need at some point Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 05:55:40 2010 @@ -278,10 +278,11 @@ (("buffer", rffi.VOIDP), ("size", Py_ssize_t))) cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) -VA_TP_LIST = {'int': lltype.Signed, - 'PyObject*': PyObject, - 'PyObject**': PyObjectP, - 'int*': rffi.INTP} +VA_TP_LIST = {} +#{'int': lltype.Signed, +# 'PyObject*': PyObject, +# 'PyObject**': PyObjectP, +# 'int*': rffi.INTP} def configure_types(): for name, TYPE in rffi_platform.configure(CConfig).iteritems(): From fijal at codespeak.net Tue Apr 6 06:02:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 06:02:28 +0200 (CEST) Subject: [pypy-svn] r73443 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406040228.25AED282B90@codespeak.net> Author: fijal Date: Tue Apr 6 06:02:26 2010 New Revision: 73443 Removed: pypy/branch/cpython-extension/pypy/module/cpyext/getargs.py pypy/branch/cpython-extension/pypy/module/cpyext/misc.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Log: simplify FUNCTIONS_C greatly (just list them) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Tue Apr 6 06:02:26 2010 @@ -46,12 +46,10 @@ import pypy.module.cpyext.listobject import pypy.module.cpyext.sequence import pypy.module.cpyext.eval -import pypy.module.cpyext.getargs import pypy.module.cpyext.import_ import pypy.module.cpyext.mapping import pypy.module.cpyext.iterator import pypy.module.cpyext.unicodeobject -import pypy.module.cpyext.misc import pypy.module.cpyext.pycobject # now that all rffi_platform.Struct types are registered, configure them Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 06:02:26 2010 @@ -207,12 +207,6 @@ return unwrapper_raise # used in 'normal' RPython code. return decorate -def register_c_function(name): - """ Register a function implemented in C as a part of API - (not yet callable from RPython) - """ - FUNCTIONS_C[name] = None - def cpython_struct(name, fields, forward=None): configname = name.replace(' ', '__') setattr(CConfig, configname, rffi_platform.Struct(name, fields)) @@ -224,7 +218,11 @@ INTERPLEVEL_API = {} FUNCTIONS = {} FUNCTIONS_STATIC = {} -FUNCTIONS_C = {} +FUNCTIONS_C = [ + 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', + 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', + 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', +] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur 'Py_None': ('PyObject*', 'space.w_None'), @@ -412,7 +410,7 @@ def build_bridge(space, rename=True): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + list(FUNCTIONS_C) + list(GLOBALS) + export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -577,7 +575,7 @@ def setup_library(space, rename=False): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + list(FUNCTIONS_C) + list(GLOBALS) + export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Tue Apr 6 06:02:26 2010 @@ -1,15 +1,12 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, register_c_function + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError -for name in ['PyModule_AddObject', 'Py_BuildValue']: - register_c_function(name) - def PyImport_AddModule(space, name): w_name = space.wrap(name) w_mod = space.wrap(Module(space, w_name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Tue Apr 6 06:02:26 2010 @@ -3,8 +3,7 @@ from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, PyStringObject, Py_ssize_t, cpython_struct, CANNOT_FAIL, build_type_checkers, - PyObjectP, PyTypeObjectPtr, - register_c_function) + PyObjectP, PyTypeObjectPtr) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef @@ -21,8 +20,6 @@ py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str -register_c_function('PyString_FromFormatV') - @cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): if char_p: From fijal at codespeak.net Tue Apr 6 06:11:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 06:11:01 +0200 (CEST) Subject: [pypy-svn] r73444 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406041101.93B20282B90@codespeak.net> Author: fijal Date: Tue Apr 6 06:10:58 2010 New Revision: 73444 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: add a skipped part that segfaults Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Tue Apr 6 06:10:58 2010 @@ -88,4 +88,6 @@ sre_compile._sre = module assert sre_compile.MAGIC == module.MAGIC import re - + skip("explodes") + m = re.search("xyz", "xyzxyz") + assert m From fijal at codespeak.net Tue Apr 6 07:26:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 07:26:40 +0200 (CEST) Subject: [pypy-svn] r73445 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100406052640.EBB07282B90@codespeak.net> Author: fijal Date: Tue Apr 6 07:26:39 2010 New Revision: 73445 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Log: one more define Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Tue Apr 6 07:26:39 2010 @@ -7,6 +7,7 @@ extern "C" { #endif +#define PyString_GET_SIZE(op) PyString_Size(op) #define PyString_AS_STRING(op) PyString_AsString(op) typedef struct { From fijal at codespeak.net Tue Apr 6 07:27:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 07:27:53 +0200 (CEST) Subject: [pypy-svn] r73446 - pypy/branch/cpython-extension/pypy/module/cpyext/src Message-ID: <20100406052753.52745282B90@codespeak.net> Author: fijal Date: Tue Apr 6 07:27:51 2010 New Revision: 73446 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Log: Try a bit harder to report unsupported cases instead of random stack corruption due to not calling va_arg correctly - does not work so far sre still segfaults. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Tue Apr 6 07:27:51 2010 @@ -135,13 +135,12 @@ PyMem_FREE(ptr); } -#if 0 //YYY static void cleanup_buffer(void *ptr) { - PyBuffer_Release((Py_buffer *) ptr); + Py_FatalError("cleanup_buffer(void *ptr) unimplemented\n"); + //PyBuffer_Release((Py_buffer *) ptr); } -#endif static int addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *)) @@ -556,8 +555,9 @@ switch (c) { -#if 0 case 'b': { /* unsigned byte -- very short int */ + Py_FatalError("'b' unimplemented for PyArg_*\n"); +#if 0 char *p = va_arg(*p_va, char *); long ival; if (float_argument_error(arg)) @@ -578,10 +578,13 @@ else *p = (unsigned char) ival; break; +#endif } case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ + Py_FatalError("'B' unimplemented for PyArg_*\n"); +#if 0 char *p = va_arg(*p_va, char *); long ival; if (float_argument_error(arg)) @@ -592,9 +595,12 @@ else *p = (unsigned char) ival; break; +#endif } case 'h': {/* signed short int */ + Py_FatalError("'h' unimplemented for PyArg_*\n"); +#if 0 short *p = va_arg(*p_va, short *); long ival; if (float_argument_error(arg)) @@ -615,10 +621,13 @@ else *p = (short) ival; break; +#endif } case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ + Py_FatalError("'H' unimplemented for PyArg_*\n"); +#if 0 unsigned short *p = va_arg(*p_va, unsigned short *); long ival; if (float_argument_error(arg)) @@ -629,8 +638,8 @@ else *p = (unsigned short) ival; break; - } #endif + } case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); long ival; @@ -653,9 +662,10 @@ *p = ival; break; } -#if 0 case 'I': { /* int sized bitfield, both signed and unsigned allowed */ + Py_FatalError("'I' unimplemented for PyArg_*\n"); +#if 0 unsigned int *p = va_arg(*p_va, unsigned int *); unsigned int ival; if (float_argument_error(arg)) @@ -666,8 +676,8 @@ else *p = ival; break; - } #endif + } case 'n': /* Py_ssize_t */ #if SIZEOF_SIZE_T != SIZEOF_LONG { @@ -696,8 +706,9 @@ break; } -#if 0 case 'k': { /* long sized bitfield */ + Py_FatalError("'k' unimplemented for PyArg_*\n"); +#if 0 unsigned long *p = va_arg(*p_va, unsigned long *); unsigned long ival; if (PyInt_Check(arg)) @@ -708,10 +719,13 @@ return converterr("integer", arg, msgbuf, bufsize); *p = ival; break; +#endif } #ifdef HAVE_LONG_LONG case 'L': {/* PY_LONG_LONG */ + Py_FatalError("'L' unimplemented for PyArg_*\n"); +#if 0 PY_LONG_LONG *p = va_arg( *p_va, PY_LONG_LONG * ); PY_LONG_LONG ival = PyLong_AsLongLong( arg ); if (ival == (PY_LONG_LONG)-1 && PyErr_Occurred() ) { @@ -720,9 +734,12 @@ *p = ival; } break; +#endif } case 'K': { /* long long sized bitfield */ + Py_FatalError("'K' unimplemented for PyArg_*\n"); +#if 0 unsigned PY_LONG_LONG *p = va_arg(*p_va, unsigned PY_LONG_LONG *); unsigned PY_LONG_LONG ival; if (PyInt_Check(arg)) @@ -733,10 +750,13 @@ return converterr("integer", arg, msgbuf, bufsize); *p = ival; break; - } -#endif - +#endif + } +#endif // HAVE_LONG_LONG + case 'f': {/* float */ + Py_FatalError("'f' unimplemented for PyArg_*\n"); +#if 0 float *p = va_arg(*p_va, float *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) @@ -744,9 +764,12 @@ else *p = (float) dval; break; +#endif } case 'd': {/* double */ + Py_FatalError("'d' unimplemented for PyArg_*\n"); +#if 0 double *p = va_arg(*p_va, double *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) @@ -754,10 +777,13 @@ else *p = dval; break; +#endif } #ifndef WITHOUT_COMPLEX case 'D': {/* complex double */ + Py_FatalError("'D' unimplemented for PyArg_*\n"); +#if 0 Py_complex *p = va_arg(*p_va, Py_complex *); Py_complex cval; cval = PyComplex_AsCComplex(arg); @@ -766,6 +792,7 @@ else *p = cval; break; +#endif } #endif /* WITHOUT_COMPLEX */ @@ -777,9 +804,10 @@ return converterr("char", arg, msgbuf, bufsize); break; } - case 's': {/* string */ if (*format == '*') { + Py_FatalError("* format unsupported for strings in PyArg_*\n"); +#if 0 Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); if (PyString_Check(arg)) { @@ -809,6 +837,7 @@ arg, msgbuf, bufsize); } format++; +#endif } else if (*format == '#') { void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -860,6 +889,8 @@ case 'z': {/* string, may be NULL (None) */ if (*format == '*') { + Py_FatalError("'*' format not supported in PyArg_*\n"); +#if 0 Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); if (arg == Py_None) @@ -891,6 +922,7 @@ arg, msgbuf, bufsize); } format++; +#endif } else if (*format == '#') { /* any buffer-like object */ void **p = (void **)va_arg(*p_va, char **); FETCH_SIZE; @@ -957,8 +989,9 @@ } break; } - case 'e': {/* encoded string */ + Py_FatalError("'e' unimplemented for PyArg_*\n"); +#if 0 char **buffer; const char *encoding; PyObject *s; @@ -971,7 +1004,7 @@ if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); #endif - + /* Get output buffer parameter: 's' (recode all objects via Unicode) or 't' (only recode non-string objects) @@ -1122,6 +1155,7 @@ } Py_DECREF(s); break; +#endif } #ifdef Py_USING_UNICODE @@ -1168,7 +1202,6 @@ break; } #endif -#endif case 'O': { /* object */ PyTypeObject *type; PyObject **p; @@ -1209,8 +1242,9 @@ break; } -#if 0 case 'w': { /* memory buffer, read-write access */ + Py_FatalError("'w' unsupported\n"); +#if 0 void **p = va_arg(*p_va, void **); void *res; PyBufferProcs *pb = arg->ob_type->tp_as_buffer; @@ -1261,9 +1295,12 @@ } } break; +#endif } case 't': { /* 8-bit character buffer, read-only access */ + Py_FatalError("'t' unsupported"); +#if 0 char **p = va_arg(*p_va, char **); PyBufferProcs *pb = arg->ob_type->tp_as_buffer; Py_ssize_t count; @@ -1298,8 +1335,8 @@ STORE_SIZE(count); } break; - } #endif + } default: return converterr("impossible", arg, msgbuf, bufsize); @@ -1378,7 +1415,7 @@ va_list va; if ((args == NULL || !PyTuple_Check(args)) || - (keywords != NULL && !PyDict_Check(keywords)) || + (keywords != NULL && !PyDict_Check(keywords)) || format == NULL || kwlist == NULL) { From fijal at codespeak.net Tue Apr 6 07:35:32 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 07:35:32 +0200 (CEST) Subject: [pypy-svn] r73447 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406053532.D5814282B90@codespeak.net> Author: fijal Date: Tue Apr 6 07:35:31 2010 New Revision: 73447 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: document a real reason why we segfault. I think 73446 was not pointless though, please revert if it was. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Tue Apr 6 07:35:31 2010 @@ -88,6 +88,6 @@ sre_compile._sre = module assert sre_compile.MAGIC == module.MAGIC import re - skip("explodes") + skip("segfaulting: unsupported METH_KEYWORDS so we get junk") m = re.search("xyz", "xyzxyz") assert m From afa at codespeak.net Tue Apr 6 11:42:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 11:42:14 +0200 (CEST) Subject: [pypy-svn] r73448 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100406094214.F3CB3282B90@codespeak.net> Author: afa Date: Tue Apr 6 11:42:12 2010 New Revision: 73448 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Fixes for Windows: crtdef.h is needed for size_t, and PyTuple_Pack needs to be exported Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 11:42:12 2010 @@ -222,6 +222,7 @@ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', + 'PyTuple_Pack', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Tue Apr 6 11:42:12 2010 @@ -23,6 +23,7 @@ # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_DATA(RTYPE) extern RTYPE #else +# include # define Py_DEPRECATED(VERSION_UNUSED) # ifdef Py_BUILD_CORE # define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE From trundle at codespeak.net Tue Apr 6 11:42:44 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Tue, 6 Apr 2010 11:42:44 +0200 (CEST) Subject: [pypy-svn] r73449 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406094244.895A1282B90@codespeak.net> Author: trundle Date: Tue Apr 6 11:42:43 2010 New Revision: 73449 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Log: Typo. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 6 11:42:43 2010 @@ -143,8 +143,8 @@ warnings module and the -W option in the command line documentation. There is no C API for warning control.""" message = rffi.charp2str(message_ptr) - if category is None: - category = space.gettypeobject(W_RuntimeWarning.typedef) + if w_category is None: + w_category = space.gettypeobject(W_RuntimeWarning.typedef) os.write(2, "WARNING: " + message + "\n") return 0 From afa at codespeak.net Tue Apr 6 13:20:47 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 13:20:47 +0200 (CEST) Subject: [pypy-svn] r73450 - in pypy/branch/cpython-extension/pypy/module/cpyext: include test Message-ID: <20100406112047.CD13B282B9C@codespeak.net> Author: afa Date: Tue Apr 6 13:20:46 2010 New Revision: 73450 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Log: Support platforms (guess which) where sizeof(wchar_t)==2. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Tue Apr 6 13:20:46 2010 @@ -14,6 +14,14 @@ #define HAVE_UNICODE #define WITHOUT_COMPLEX +/* PyPy supposes Py_UNICODE == wchar_t */ +#define HAVE_USABLE_WCHAR_T 1 +#ifndef _WIN32 +#define Py_UNICODE_SIZE 4 +#else +#define Py_UNICODE_SIZE 2 +#endif + /* Compat stuff */ #ifndef _WIN32 # include @@ -23,6 +31,7 @@ # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_DATA(RTYPE) extern RTYPE #else +# define MS_WIN32 1 # include # define Py_DEPRECATED(VERSION_UNUSED) # ifdef Py_BUILD_CORE @@ -31,6 +40,7 @@ # define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE # endif #endif + #define Py_ssize_t long #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Tue Apr 6 13:20:46 2010 @@ -13,10 +13,14 @@ } PyUnicodeObject; -//XXX -typedef unsigned int Py_UCS4; -// pypy only supports only UCS4 +typedef unsigned int Py_UCS4; +#ifdef HAVE_USABLE_WCHAR_T +#define PY_UNICODE_TYPE wchar_t +#elif Py_UNICODE_SIZE == 4 #define PY_UNICODE_TYPE Py_UCS4 +#else +#define PY_UNICODE_TYPE unsigned short +#endif typedef PY_UNICODE_TYPE Py_UNICODE; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Tue Apr 6 13:20:46 2010 @@ -1,13 +1,13 @@ # encoding: iso-8859-15 from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.unicodeobject import Py_UNICODE -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, lltype class TestUnicode(BaseApiTest): def test_unicodeobject(self, space, api): assert api.PyUnicode_GET_SIZE(space.wrap(u'sp?m')) == 4 - # XXX assuming UCS-4 - assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 16 + unichar = rffi.sizeof(lltype.UniChar) + assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 4 * unichar def test_AS_DATA(self, space, api): word = space.wrap(u'spam') From afa at codespeak.net Tue Apr 6 13:52:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 13:52:02 +0200 (CEST) Subject: [pypy-svn] r73451 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100406115202.179D7282B9C@codespeak.net> Author: afa Date: Tue Apr 6 13:52:00 2010 New Revision: 73451 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: fix the Py_UNICODE type. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Tue Apr 6 13:52:00 2010 @@ -6,7 +6,7 @@ class TestUnicode(BaseApiTest): def test_unicodeobject(self, space, api): assert api.PyUnicode_GET_SIZE(space.wrap(u'sp?m')) == 4 - unichar = rffi.sizeof(lltype.UniChar) + unichar = rffi.sizeof(Py_UNICODE) assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 4 * unichar def test_AS_DATA(self, space, api): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 6 13:52:00 2010 @@ -7,8 +7,7 @@ PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode") -# XXX -Py_UNICODE = rffi.UINT +Py_UNICODE = lltype.UniChar @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, w_ch): From xoraxax at codespeak.net Tue Apr 6 15:53:50 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 15:53:50 +0200 (CEST) Subject: [pypy-svn] r73452 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406135350.AA88F282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 15:53:49 2010 New Revision: 73452 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: If a name is given, create a func_with_new_name of the func in decorate(). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 15:53:49 2010 @@ -25,6 +25,7 @@ from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException +from pypy.tool.sourcetools import func_with_new_name DEBUG_WRAPPER = False @@ -132,12 +133,13 @@ error = rffi.cast(restype, error) def decorate(func): - api_function = ApiFunction(argtypes, restype, func, borrowed, error) - func.api_func = api_function if name is None: func_name = func.func_name else: func_name = name + func = func_with_new_name(func, name) + api_function = ApiFunction(argtypes, restype, func, borrowed, error) + func.api_func = api_function if error is _NOT_SPECIFIED: raise ValueError("function %s has no return value for exceptions" From xoraxax at codespeak.net Tue Apr 6 15:54:20 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 15:54:20 +0200 (CEST) Subject: [pypy-svn] r73453 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406135420.58106282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 15:54:18 2010 New Revision: 73453 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Make from_ref a bit more robust against invalid pointers. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Tue Apr 6 15:54:18 2010 @@ -122,7 +122,7 @@ return w_str -def from_ref(space, ref): +def from_ref(space, ref, recurse=False): from pypy.module.cpyext.typeobject import PyPyType_Ready assert lltype.typeOf(ref) == PyObject if not ref: @@ -132,19 +132,18 @@ try: w_obj = state.py_objects_r2w[ptr] except KeyError: + if recurse: + raise InvalidPointerException(str(ref)) ref_type = rffi.cast(PyObject, ref.c_ob_type) if ref != ref_type: - w_type = from_ref(space, ref_type) + w_type = from_ref(space, ref_type, True) if space.is_w(w_type, space.w_str): - return force_string(space, ref) + return force_string(space, ref, True) elif space.is_w(w_type, space.w_type): PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) - return from_ref(space, ref) + return from_ref(space, ref, True) else: - msg = "" - if not we_are_translated(): - msg = "Got invalid reference to a PyObject: %r" % (ref, ) - raise InvalidPointerException(msg) + raise InvalidPointerException(str(ref)) else: raise InvalidPointerException("This should never happen") return w_obj From xoraxax at codespeak.net Tue Apr 6 15:55:03 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 15:55:03 +0200 (CEST) Subject: [pypy-svn] r73454 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406135503.E8568282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 15:55:02 2010 New Revision: 73454 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Test is passing (but leaking)! Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Tue Apr 6 15:55:02 2010 @@ -88,6 +88,9 @@ sre_compile._sre = module assert sre_compile.MAGIC == module.MAGIC import re - skip("segfaulting: unsupported METH_KEYWORDS so we get junk") - m = re.search("xyz", "xyzxyz") + import time + s = u"Foo " * 1000 + u"Bar" + prog = re.compile(ur"Foo.*Bar") + assert prog.match(s) + m = re.search(u"xyz", u"xyzxyz") assert m From xoraxax at codespeak.net Tue Apr 6 15:55:26 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 15:55:26 +0200 (CEST) Subject: [pypy-svn] r73455 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406135526.B6D0036C225@codespeak.net> Author: xoraxax Date: Tue Apr 6 15:55:25 2010 New Revision: 73455 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: Implement METH_KEYWORD calls. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Tue Apr 6 15:55:25 2010 @@ -9,13 +9,14 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ - cpython_struct + cpython_struct, METH_KEYWORDS from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) +PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject], PyObject)) PyMethodDef = cpython_struct( 'PyMethodDef', @@ -32,11 +33,20 @@ self.w_self = w_self self.doc = doc - def call(self, space, w_self, args_tuple): + def call(self, space, w_self, w_args, w_kw): # Call the C function if w_self is None: w_self = self.w_self - return generic_cpy_call(space, self.ml.c_ml_meth, w_self, args_tuple) + flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) + if space.is_true(w_kw) and not flags & METH_KEYWORDS: + raise OperationError(space.w_TypeError, + space.wrap(rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) + # XXX support METH_NOARGS, METH_O + if flags & METH_KEYWORDS: + func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) + return generic_cpy_call(space, func, w_self, w_args, w_kw) + else: + return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_args) class W_PyCMethodObject(W_PyCFunctionObject): @@ -99,22 +109,22 @@ self = space.interp_w(W_PyCFunctionObject, w_self) args_w, kw_w = __args__.unpack() w_args = space.newtuple(args_w) - if kw_w: - raise OperationError(space.w_TypeError, - space.wrap("keywords not yet supported")) - ret = self.call(space, None, w_args) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.wrap(key), w_obj) + ret = self.call(space, None, w_args, w_kw) return ret @unwrap_spec(ObjSpace, W_Root, Arguments) def cmethod_descr_call(space, w_self, __args__): self = space.interp_w(W_PyCFunctionObject, w_self) args_w, kw_w = __args__.unpack() - w_instance = args_w[0] + w_instance = args_w[0] # XXX typecheck missing w_args = space.newtuple(args_w[1:]) - if kw_w: - raise OperationError(space.w_TypeError, - space.wrap("keywords not yet supported")) - ret = self.call(space, w_instance, w_args) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.wrap(key), w_obj) + ret = self.call(space, w_instance, w_args, w_kw) return ret def cmethod_descr_get(space, w_function, w_obj, w_cls=None): From xoraxax at codespeak.net Tue Apr 6 16:02:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 16:02:25 +0200 (CEST) Subject: [pypy-svn] r73456 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406140225.3938D282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 16:02:23 2010 New Revision: 73456 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: The leaks were caches in re.py. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Tue Apr 6 16:02:23 2010 @@ -94,3 +94,5 @@ assert prog.match(s) m = re.search(u"xyz", u"xyzxyz") assert m + re._cache.clear() + re._cache_repl.clear() From xoraxax at codespeak.net Tue Apr 6 16:25:40 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 16:25:40 +0200 (CEST) Subject: [pypy-svn] r73457 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406142540.D0D8A282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 16:25:39 2010 New Revision: 73457 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Running tests is overrated. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Tue Apr 6 16:25:39 2010 @@ -138,7 +138,7 @@ if ref != ref_type: w_type = from_ref(space, ref_type, True) if space.is_w(w_type, space.w_str): - return force_string(space, ref, True) + return force_string(space, ref) elif space.is_w(w_type, space.w_type): PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) return from_ref(space, ref, True) From xoraxax at codespeak.net Tue Apr 6 16:57:07 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 16:57:07 +0200 (CEST) Subject: [pypy-svn] r73458 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406145707.680D3282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 16:57:05 2010 New Revision: 73458 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Also declare PyErr_Format. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 16:57:05 2010 @@ -224,7 +224,7 @@ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', - 'PyTuple_Pack', + 'PyTuple_Pack', 'PyErr_Format', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur From afa at codespeak.net Tue Apr 6 17:26:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 17:26:53 +0200 (CEST) Subject: [pypy-svn] r73459 - in pypy/branch/cpython-extension/pypy/rpython/lltypesystem: . test Message-ID: <20100406152653.63C97282B9C@codespeak.net> Author: afa Date: Tue Apr 6 17:26:51 2010 New Revision: 73459 Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: ll2ctypes should accept all arguments passed to the call. This is needed by METH_KEYWORDS functions, which take 3 PyObjects*, but are cast to a PyCFunction taking 2 PyObjects*. Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/ll2ctypes.py Tue Apr 6 17:26:51 2010 @@ -964,7 +964,7 @@ def invoke_via_ctypes(*argvalues): global _callback_exc_info cargs = [] - for i in range(len(FUNCTYPE.ARGS)): + for i in range(len(argvalues)): if i not in void_arguments: cvalue = lltype2ctypes(argvalues[i]) if i in container_arguments: Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Tue Apr 6 17:26:51 2010 @@ -432,6 +432,29 @@ rffi.free_charp(p) assert not ALLOCATED # detects memory leaks in the test + def test_funcptr_cast(self): + eci = ExternalCompilationInfo( + separate_module_sources=[""" + long mul(long x, long y) { return x*y; } + long(*get_mul(long x)) () { return &mul; } + """], + export_symbols=['get_mul']) + get_mul = rffi.llexternal( + 'get_mul', [], + lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)), + compilation_info=eci) + # This call returns a pointer to a function taking one argument + funcptr = get_mul() + # cast it to the "real" function type + FUNCTYPE2 = lltype.FuncType([lltype.Signed, lltype.Signed], + lltype.Signed) + cmul = rffi.cast(lltype.Ptr(FUNCTYPE2), funcptr) + # and it can be called with the expected number of arguments + res = cmul(41, 42) + assert res == 41 * 42 + raises(TypeError, cmul, 41) + raises(TypeError, cmul, 41, 42, 43) + def test_qsort(self): CMPFUNC = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], rffi.INT) qsort = rffi.llexternal('qsort', [rffi.VOIDP, From fijal at codespeak.net Tue Apr 6 17:32:25 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 17:32:25 +0200 (CEST) Subject: [pypy-svn] r73460 - pypy/branch/cpython-extension/pypy/module/cpyext/src Message-ID: <20100406153225.A5B51282B9C@codespeak.net> Author: fijal Date: Tue Apr 6 17:32:24 2010 New Revision: 73460 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Log: #if 0 the cleanup_buffer function Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/getargs.c Tue Apr 6 17:32:24 2010 @@ -135,12 +135,13 @@ PyMem_FREE(ptr); } +#if 0 static void cleanup_buffer(void *ptr) { - Py_FatalError("cleanup_buffer(void *ptr) unimplemented\n"); - //PyBuffer_Release((Py_buffer *) ptr); + PyBuffer_Release((Py_buffer *) ptr); } +#endif static int addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *)) @@ -1219,7 +1220,7 @@ inquiry pred = va_arg(*p_va, inquiry); p = va_arg(*p_va, PyObject **); format++; - if ((*pred)(arg)) + if ((*pred)(arg)) *p = arg; else return converterr("(unspecified)", From afa at codespeak.net Tue Apr 6 18:33:47 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 18:33:47 +0200 (CEST) Subject: [pypy-svn] r73461 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406163347.24B8D282B9C@codespeak.net> Author: afa Date: Tue Apr 6 18:33:44 2010 New Revision: 73461 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Various translation fixes Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 18:33:44 2010 @@ -169,7 +169,7 @@ arg = input_arg elif ARG is PyObject and is_wrapped: # convert to a wrapped object - if rffi._isllptr(input_arg): + if input_arg and rffi._isllptr(input_arg): arg = from_ref(space, input_arg) else: arg = input_arg @@ -324,8 +324,7 @@ print >>sys.stderr, callable, for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] - if (typ is PyObject and - is_wrapped): + if typ is PyObject and is_wrapped: if arg: arg_conv = from_ref(space, arg) else: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Tue Apr 6 18:33:44 2010 @@ -19,7 +19,7 @@ def __init__(self, space, voidp): W_PyCObject.__init__(self, space) self.voidp = voidp - self.pyo = lltype.nullptr(PyObject.TO) + self.pyo = lltype.nullptr(PyCObject.TO) def set_pycobject(self, pyo): self.pyo = pyo @@ -33,6 +33,7 @@ """Create a PyCObject from the void * cobj. The destr function will be called when the object is reclaimed, unless it is NULL.""" w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj)) + assert isinstance(w_pycobject, W_PyCObjectFromVoidPtr) pyo = make_ref(space, w_pycobject) pycobject = rffi.cast(PyCObject, pyo) w_pycobject.set_pycobject(pycobject) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 6 18:33:44 2010 @@ -156,6 +156,6 @@ calling PyErr_WarnEx() with a stacklevel of 1. Deprecated; use PyErr_WarnEx() instead.""" - return PyErr_WarnEx(w_category, message, 1) + return PyErr_WarnEx(space, w_category, message, 1) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 6 18:33:44 2010 @@ -10,49 +10,49 @@ Py_UNICODE = lltype.UniChar @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISSPACE(space, w_ch): +def Py_UNICODE_ISSPACE(space, ch): """Return 1 or 0 depending on whether ch is a whitespace character.""" - return unicodedb.isspace(w_ch) + return unicodedb.isspace(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISALNUM(space, w_ch): +def Py_UNICODE_ISALNUM(space, ch): """Return 1 or 0 depending on whether ch is an alphanumeric character.""" - return unicodedb.isalnum(w_ch) + return unicodedb.isalnum(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISLINEBREAK(space, w_ch): +def Py_UNICODE_ISLINEBREAK(space, ch): """Return 1 or 0 depending on whether ch is a linebreak character.""" - return unicodedb.islinebreak(w_ch) + return unicodedb.islinebreak(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISSPACE(space, w_ch): +def Py_UNICODE_ISSPACE(space, ch): """Return 1 or 0 depending on whether ch is a whitespace character.""" - return unicodedb.isspace(w_ch) + return unicodedb.isspace(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISALNUM(space, w_ch): +def Py_UNICODE_ISALNUM(space, ch): """Return 1 or 0 depending on whether ch is an alphanumeric character.""" - return unicodedb.isalnum(w_ch) + return unicodedb.isalnum(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISDECIMAL(space, w_ch): +def Py_UNICODE_ISDECIMAL(space, ch): """Return 1 or 0 depending on whether ch is a decimal character.""" - return unicodedb.isdecimal(w_ch) + return unicodedb.isdecimal(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISLOWER(space, w_ch): +def Py_UNICODE_ISLOWER(space, ch): """Return 1 or 0 depending on whether ch is a lowercase character.""" - return unicodedb.islower(w_ch) + return unicodedb.islower(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISUPPER(space, w_ch): +def Py_UNICODE_ISUPPER(space, ch): """Return 1 or 0 depending on whether ch is an uppercase character.""" - return unicodedb.isupper(w_ch) + return unicodedb.isupper(ord(ch)) @cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL) -def Py_UNICODE_TOLOWER(space, w_ch): +def Py_UNICODE_TOLOWER(space, ch): """Return the character ch converted to lower case.""" - return unicodedb.tolower(w_ch) + return unicodedb.tolower(ord(ch)) @cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyUnicode_AS_DATA(space, ref): @@ -67,10 +67,10 @@ return rffi.cast(rffi.CCHARP, ref_unicode.c_buffer) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) -def PyUnicode_GET_DATA_SIZE(space, obj): +def PyUnicode_GET_DATA_SIZE(space, w_obj): """Return the size of the object's internal buffer in bytes. o has to be a PyUnicodeObject (not checked).""" - return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, obj) + return rffi.sizeof(lltype.UniChar) * PyUnicode_GET_SIZE(space, w_obj) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyUnicode_GET_SIZE(space, w_obj): From afa at codespeak.net Tue Apr 6 18:49:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 18:49:43 +0200 (CEST) Subject: [pypy-svn] r73462 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100406164943.366CE282B9C@codespeak.net> Author: afa Date: Tue Apr 6 18:49:41 2010 New Revision: 73462 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Run the tests, and fix. At RPython level, Py_UNICODE is a single unicode char. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 18:49:41 2010 @@ -169,10 +169,12 @@ arg = input_arg elif ARG is PyObject and is_wrapped: # convert to a wrapped object - if input_arg and rffi._isllptr(input_arg): - arg = from_ref(space, input_arg) - else: + if input_arg is None: + arg = input_arg + elif isinstance(input_arg, W_Root): arg = input_arg + else: + arg = from_ref(space, input_arg) else: arg = input_arg newargs += (arg, ) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Tue Apr 6 18:49:41 2010 @@ -22,23 +22,23 @@ 0x2009, 0x200a, #0x200b is in Other_Default_Ignorable_Code_Point in 4.1.0 0x2028, 0x2029, 0x202f, 0x205f, 0x3000]: - assert api.Py_UNICODE_ISSPACE(char) - assert not api.Py_UNICODE_ISSPACE(ord(u'a')) + assert api.Py_UNICODE_ISSPACE(unichr(char)) + assert not api.Py_UNICODE_ISSPACE(u'a') - assert api.Py_UNICODE_ISDECIMAL(ord(u'\u0660')) - assert not api.Py_UNICODE_ISDECIMAL(ord(u'a')) + assert api.Py_UNICODE_ISDECIMAL(u'\u0660') + assert not api.Py_UNICODE_ISDECIMAL(u'a') for char in [0x0a, 0x0d, 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029]: - assert api.Py_UNICODE_ISLINEBREAK(char) + assert api.Py_UNICODE_ISLINEBREAK(unichr(char)) - assert api.Py_UNICODE_ISLOWER(ord(u'?')) - assert not api.Py_UNICODE_ISUPPER(ord(u'?')) - assert api.Py_UNICODE_ISLOWER(ord(u'a')) - assert not api.Py_UNICODE_ISUPPER(ord(u'a')) - assert not api.Py_UNICODE_ISLOWER(ord(u'?')) - assert api.Py_UNICODE_ISUPPER(ord(u'?')) + assert api.Py_UNICODE_ISLOWER(u'?') + assert not api.Py_UNICODE_ISUPPER(u'?') + assert api.Py_UNICODE_ISLOWER(u'a') + assert not api.Py_UNICODE_ISUPPER(u'a') + assert not api.Py_UNICODE_ISLOWER(u'?') + assert api.Py_UNICODE_ISUPPER(u'?') def test_TOLOWER(self, space, api): - assert api.Py_UNICODE_TOLOWER(ord(u'?')) == ord(u'?') - assert api.Py_UNICODE_TOLOWER(ord(u'?')) == ord(u'?') + assert api.Py_UNICODE_TOLOWER(u'?') == u'?' + assert api.Py_UNICODE_TOLOWER(u'?') == u'?' Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 6 18:49:41 2010 @@ -52,7 +52,7 @@ @cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL) def Py_UNICODE_TOLOWER(space, ch): """Return the character ch converted to lower case.""" - return unicodedb.tolower(ord(ch)) + return unichr(unicodedb.tolower(ord(ch))) @cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyUnicode_AS_DATA(space, ref): From afa at codespeak.net Tue Apr 6 19:41:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 19:41:51 +0200 (CEST) Subject: [pypy-svn] r73463 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100406174151.F0B21282B9C@codespeak.net> Author: afa Date: Tue Apr 6 19:41:50 2010 New Revision: 73463 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: translation fixes Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 6 19:41:50 2010 @@ -77,12 +77,12 @@ so an object allocation function can write return PyErr_NoMemory(); when it runs out of memory. Return value: always NULL.""" - PyErr_SetNone(space.w_MemoryError) + PyErr_SetNone(space, space.w_MemoryError) @cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL) def PyErr_SetNone(space, w_type): """This is a shorthand for PyErr_SetObject(type, Py_None).""" - PyErr_SetObject(w_type, space.w_None) + PyErr_SetObject(space, w_type, space.w_None) @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Tue Apr 6 19:41:50 2010 @@ -54,6 +54,16 @@ api.PyErr_Clear() + def test_SetNone(self, space, api): + api.PyErr_SetNone(space.w_KeyError) + state = space.fromcache(State) + assert space.eq_w(state.exc_type, space.w_KeyError) + assert space.eq_w(state.exc_value, space.w_None) + api.PyErr_Clear() + + api.PyErr_NoMemory() + assert space.eq_w(state.exc_type, space.w_MemoryError) + api.PyErr_Clear() class AppTestFetch(AppTestCpythonExtensionBase): From afa at codespeak.net Tue Apr 6 19:51:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 19:51:45 +0200 (CEST) Subject: [pypy-svn] r73464 - in pypy/branch/cpython-extension/pypy: interpreter module/cpyext Message-ID: <20100406175145.5F332282B9C@codespeak.net> Author: afa Date: Tue Apr 6 19:51:43 2010 New Revision: 73464 Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix translation of the GetSetProperty objects Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/typedef.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/typedef.py Tue Apr 6 19:51:43 2010 @@ -301,7 +301,7 @@ # ____________________________________________________________ - at specialize.arg(0, 2) + at specialize.arg(0) def make_descr_typecheck_wrapper(tag, func, extraargs=(), cls=None, use_closure=False): if func is None: @@ -399,12 +399,12 @@ def __init__(self, fget, fset=None, fdel=None, doc=None, cls=None, use_closure=False, tag=None): objclass_getter, cls = make_objclass_getter(tag, fget, cls) - fget = make_descr_typecheck_wrapper(tag, fget, cls=cls, - use_closure=use_closure) - fset = make_descr_typecheck_wrapper(tag, fset, ('w_value',), cls=cls, - use_closure=use_closure) - fdel = make_descr_typecheck_wrapper(tag, fdel, cls=cls, - use_closure=use_closure) + fget = make_descr_typecheck_wrapper((tag, 0), fget, + cls=cls, use_closure=use_closure) + fset = make_descr_typecheck_wrapper((tag, 1), fset, ('w_value',), + cls=cls, use_closure=use_closure) + fdel = make_descr_typecheck_wrapper((tag, 2), fdel, + cls=cls, use_closure=use_closure) self.fget = fget self.fset = fset self.fdel = fdel Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 19:51:43 2010 @@ -51,7 +51,7 @@ if getset.c_set: set = GettersAndSetters.setter.im_func GetSetProperty.__init__(self, get, set, None, doc, - cls=None, use_closure=True, # XXX cls? + cls=None, use_closure=True, tag="cpyext_1") def PyDescr_NewGetSet(space, getset, pto): @@ -70,7 +70,7 @@ if not (flags & structmemberdefs.READONLY): set = GettersAndSetters.member_setter.im_func GetSetProperty.__init__(self, get, set, del_, doc, - cls=None, use_closure=True, # XXX cls? + cls=None, use_closure=True, tag="cpyext_2") def convert_getset_defs(space, dict_w, getsets, pto): From xoraxax at codespeak.net Tue Apr 6 22:05:23 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 6 Apr 2010 22:05:23 +0200 (CEST) Subject: [pypy-svn] r73466 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100406200523.C2C89282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 6 22:05:21 2010 New Revision: 73466 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: A failing test for amaury. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Tue Apr 6 22:05:21 2010 @@ -81,6 +81,7 @@ def baz(self): return self assert fuu2(u"abc").baz().escape() + raises(TypeError, module.fooType.object_member.__get__, 1) def test_sre(self): module = self.import_module(name='_sre') From fijal at codespeak.net Tue Apr 6 22:16:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Apr 2010 22:16:20 +0200 (CEST) Subject: [pypy-svn] r73469 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100406201620.BF423282B9C@codespeak.net> Author: fijal Date: Tue Apr 6 22:16:19 2010 New Revision: 73469 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Log: Implement the first part of buffer protocol - the simplest one getsegcount Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 6 22:16:19 2010 @@ -260,9 +260,21 @@ # So we need a forward and backward mapping in our State instance PyObjectStruct = lltype.ForwardReference() PyObject = lltype.Ptr(PyObjectStruct) +PyBufferProcs = lltype.ForwardReference() PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyTypeObjectPtr)) +def F(ARGS, RESULT=lltype.Signed): + return lltype.Ptr(lltype.FuncType(ARGS, RESULT)) +PyBufferProcsFields = ( + ("bf_getreadbuffer", F([PyObject, lltype.Signed, rffi.VOIDPP])), + ("bf_getwritebuffer", F([PyObject, lltype.Signed, rffi.VOIDPP])), + ("bf_getsegcount", F([PyObject, rffi.INTP])), + ("bf_getcharbuffer", F([PyObject, lltype.Signed, rffi.CCHARPP])), +# we don't support new buffer interface for now + ("bf_getbuffer", rffi.VOIDP), + ("bf_releasebuffer", rffi.VOIDP)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('struct _object', PyObjectFields, PyObjectStruct) +cpython_struct('PyBufferProcs', PyBufferProcsFields, PyBufferProcs) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) PyVarObject = lltype.Ptr(PyVarObjectStruct) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Tue Apr 6 22:16:19 2010 @@ -183,7 +183,7 @@ writebufferproc bf_getwritebuffer; segcountproc bf_getsegcount; charbufferproc bf_getcharbuffer; - getbufferproc bf_getbuffer; + getbufferproc bf_getbuffer; releasebufferproc bf_releasebuffer; } PyBufferProcs; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Tue Apr 6 22:16:19 2010 @@ -149,3 +149,15 @@ assert py_str.c_buffer[10] == '\x00' Py_DecRef(space, ar[0]) lltype.free(ar, flavor='raw') + + def test_string_buffer(self, space, api): + py_str = new_empty_str(space, 10) + c_buf = py_str.c_ob_type.c_tp_as_buffer + assert c_buf + py_obj = rffi.cast(PyObject, py_str) + assert c_buf.c_bf_getsegcount(py_obj, lltype.nullptr(rffi.INTP.TO)) == 1 + ref = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + assert c_buf.c_bf_getsegcount(py_obj, ref) == 1 + assert ref[0] == 10 + lltype.free(ref, flavor='raw') + Py_DecRef(space, py_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 22:16:19 2010 @@ -16,7 +16,7 @@ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - PyUnicodeObject, CANNOT_FAIL + PyUnicodeObject, CANNOT_FAIL, PyBufferProcs from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod @@ -307,6 +307,20 @@ # hopefully this does not clash with the memory model assumed in # extension modules + at cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, + error=CANNOT_FAIL) +def str_segcount(space, w_obj, ref): + if ref: + ref[0] = space.int_w(space.len(w_obj)) + return 1 + +def get_helper(space, func): + return llhelper(func.api_func.functype, func.api_func.get_wrapper(space)) + +def setup_string_buffer_procs(space, pto): + c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) + c_buf.c_bf_getsegcount = get_helper(space, str_segcount) + pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) def string_dealloc(space, obj): @@ -338,6 +352,9 @@ base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython + if obj_pto.c_tp_as_buffer: + lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') + # this does not have a refcount if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: Py_DecRef(space, base_pyo) lltype.free(obj_pto.c_tp_name, flavor="raw") @@ -369,6 +386,8 @@ elif space.is_w(w_type, space.w_str): pto.c_tp_dealloc = llhelper(string_dealloc.api_func.functype, string_dealloc.api_func.get_wrapper(space)) + # buffer protocol + setup_string_buffer_procs(space, pto) elif space.is_w(w_type, space.w_unicode): pto.c_tp_dealloc = llhelper(unicode_dealloc.api_func.functype, unicode_dealloc.api_func.get_wrapper(space)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Tue Apr 6 22:16:19 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ - PyTypeObject, PyTypeObjectPtr + PyTypeObject, PyTypeObjectPtr, PyBufferProcs from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef @@ -12,10 +12,6 @@ P, FT, PyO = Ptr, FuncType, PyObject PyOPtr = Ptr(lltype.Array(PyO, hints={'nolength': True})) - -# XXX -PyBufferProcs = rffi.VOIDP.TO - freefunc = P(FT([rffi.VOIDP_real], Void)) destructor = P(FT([PyO], Void)) printfunc = P(FT([PyO, rffi.VOIDP_real, rffi.INT_real], rffi.INT)) From benjamin at codespeak.net Tue Apr 6 22:36:33 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 6 Apr 2010 22:36:33 +0200 (CEST) Subject: [pypy-svn] r73470 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406203633.D5072282B9C@codespeak.net> Author: benjamin Date: Tue Apr 6 22:36:32 2010 New Revision: 73470 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: another 64 bit cast Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 22:36:32 2010 @@ -311,7 +311,7 @@ error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): if ref: - ref[0] = space.int_w(space.len(w_obj)) + ref[0] = rffi.cast(rffi.INT, space.int_w(space.len(w_obj))) return 1 def get_helper(space, func): From antoine at codespeak.net Tue Apr 6 23:35:36 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Tue, 6 Apr 2010 23:35:36 +0200 (CEST) Subject: [pypy-svn] r73471 - in pypy/branch/decouple-host-opcodes/pypy: interpreter tool translator/goal Message-ID: <20100406213536.D73C6282B9C@codespeak.net> Author: antoine Date: Tue Apr 6 23:35:35 2010 New Revision: 73471 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Log: Translation finally works (after much bothering with unrolling_iterable's children) Next step would be to add 2.7 support (which will probably unveil the bugs :-() Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Tue Apr 6 23:35:35 2010 @@ -14,9 +14,8 @@ from pypy.rlib import jit, rstackovf from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.tool.stdlib_opcode import (unrolling_opcode_descs, - HAVE_ARGUMENT, host_HAVE_ARGUMENT, opcodedesc, host_opcodedesc, - opcode_method_names, host_opcode_method_names, ) +from pypy.tool.stdlib_opcode import ( + bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) def unaryoperation(operationname): """NOT_RPYTHON""" @@ -164,6 +163,9 @@ @jit.unroll_safe def dispatch_bytecode(self, co_code, next_instr, ec): space = self.space + cls = self.__class__ + # PyFrame is really an abstract class + assert self.__class__ is not pyframe.PyFrame while True: self.last_instr = intmask(next_instr) if not jit.we_are_jitted(): @@ -228,15 +230,19 @@ if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, next_instr, ec) - if we_are_translated(): + if 1 or we_are_translated(): from pypy.rlib import rstack # for resume points - for opdesc in unrolling_opcode_descs: + for opdesc in unrolling_all_opcode_descs: # static checks to skip this whole case if necessary + if opdesc.bytecode_spec is not self.bytecode_spec: + continue if not opdesc.is_enabled(space): continue - if not hasattr(pyframe.PyFrame, opdesc.methodname): - continue # e.g. for JUMP_ABSOLUTE, implemented above + if opdesc.methodname in ( + 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'END_FINALLY', 'JUMP_ABSOLUTE'): + continue # opcodes implemented above if opcode == opdesc.index: # dispatch to the opcode method @@ -1001,9 +1007,11 @@ class __extend__(pyframe.PyPyFrame): - opcode_method_names = opcode_method_names - opcodedesc = opcodedesc - HAVE_ARGUMENT = HAVE_ARGUMENT + bytecode_spec = bytecode_spec + opcode_method_names = bytecode_spec.method_names + opcodedesc = bytecode_spec.opcodedesc + opdescmap = bytecode_spec.opdescmap + HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT def BUILD_MAP(self, itemcount, next_instr): if itemcount != 0: @@ -1017,9 +1025,11 @@ host_version_info = sys.version_info class __extend__(pyframe.HostPyFrame): - opcode_method_names = host_opcode_method_names - opcodedesc = host_opcodedesc - HAVE_ARGUMENT = host_HAVE_ARGUMENT + bytecode_spec = host_bytecode_spec + opcode_method_names = host_bytecode_spec.method_names + opcodedesc = host_bytecode_spec.opcodedesc + opdescmap = host_bytecode_spec.opdescmap + HAVE_ARGUMENT = host_bytecode_spec.HAVE_ARGUMENT def BUILD_MAP(self, itemcount, next_instr): if host_version_info >= (2, 6): Modified: pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/tool/stdlib_opcode.py Tue Apr 6 23:35:35 2010 @@ -1,108 +1,131 @@ +""" +Opcodes PyPy compiles Python source to. +Also gives access to opcodes of the host Python PyPy was bootstrapped with +(module attributes with the `host_` prefix). +""" + # load opcode.py as pythonopcode from our own lib __all__ = ['opmap', 'opname', 'HAVE_ARGUMENT', 'hasconst', 'hasname', 'hasjrel', 'hasjabs', 'haslocal', 'hascompare', 'hasfree', 'cmp_op'] -from opcode import ( - opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) - -def load_opcode(): - import py - opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') - d = {} - execfile(str(opcode_path), d) - return d - -opcode_dict = load_opcode() -del load_opcode - -# copy some stuff from opcode.py directly into our globals -for name in __all__: - if name in opcode_dict: - globals()[name] = opcode_dict[name] -globals().update(opmap) -SLICE = opmap["SLICE+0"] -STORE_SLICE = opmap["STORE_SLICE+0"] -DELETE_SLICE = opmap["DELETE_SLICE+0"] - -def make_method_names(opmap): - tbl = ['MISSING_OPCODE'] * 256 - for name, index in opmap.items(): - tbl[index] = name.replace('+', '_') - return tbl - -opcode_method_names = make_method_names(opmap) -host_opcode_method_names = make_method_names(host_opmap) -#print ( - #set(enumerate(opcode_method_names)) ^ set(enumerate(host_opcode_method_names)) -#) -del make_method_names - # ____________________________________________________________ # RPython-friendly helpers and structures -from pypy.rlib.unroll import unrolling_iterable +class _BaseOpcodeDesc(object): + def __init__(self, bytecode_spec, name, index, methodname): + self.bytecode_spec = bytecode_spec + self.name = name + self.methodname = methodname + self.index = index + self.hasarg = index >= self.HAVE_ARGUMENT + + def _freeze_(self): + return True + + def is_enabled(self, space): + """Check if the opcode should be enabled in the space's configuration. + (Returns True for all standard opcodes.)""" + opt = space.config.objspace.opcodes + return getattr(opt, self.name, True) + is_enabled._annspecialcase_ = 'specialize:memo' + + # for predictable results, we try to order opcodes most-used-first + opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] + + def sortkey(self): + try: + i = self.opcodeorder.index(self.index) + except ValueError: + i = 1000000 + return i, self.index + + def __cmp__(self, other): + return (cmp(self.__class__, other.__class__) or + cmp(self.sortkey(), other.sortkey())) + + def __str__(self): + return "" % (self.index, self.name, id(self)) + + __repr__ = __str__ -def make_opcode_desc(HAVE_ARGUMENT): - class OpcodeDesc(object): - def __init__(self, name, index): - self.name = name - self.methodname = opcode_method_names[index] - self.index = index - self.hasarg = index >= HAVE_ARGUMENT - - def _freeze_(self): - return True - - def is_enabled(self, space): - """Check if the opcode should be enabled in the space's configuration. - (Returns True for all standard opcodes.)""" - opt = space.config.objspace.opcodes - return getattr(opt, self.name, True) - is_enabled._annspecialcase_ = 'specialize:memo' - - # for predictable results, we try to order opcodes most-used-first - opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] - - def sortkey(self): - try: - i = self.opcodeorder.index(self.index) - except ValueError: - i = 1000000 - return i, self.index - - def __cmp__(self, other): - return (cmp(self.__class__, other.__class__) or - cmp(self.sortkey(), other.sortkey())) +class _baseopcodedesc: + """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs.""" + pass - return OpcodeDesc -OpcodeDesc = make_opcode_desc(HAVE_ARGUMENT) -HostOpcodeDesc = make_opcode_desc(host_HAVE_ARGUMENT) +class BytecodeSpec(object): + """A bunch of mappings describing a bytecode instruction set.""" -opdescmap = {} + def __init__(self, name, opmap, HAVE_ARGUMENT): + """NOT_RPYTHON.""" + class OpcodeDesc(_BaseOpcodeDesc): + HAVE_ARGUMENT = HAVE_ARGUMENT + class opcodedesc(_baseopcodedesc): + """A namespace mapping OPCODE_NAME to OpcodeDescs.""" + + self.name = name + self.OpcodeDesc = OpcodeDesc + self.opcodedesc = opcodedesc + self.HAVE_ARGUMENT = HAVE_ARGUMENT + # opname -> opcode + self.opmap = opmap + # opcode -> method name + self.method_names = tbl = ['MISSING_OPCODE'] * 256 + # opcode -> opdesc + self.opdescmap = {} + for name, index in opmap.items(): + tbl[index] = methodname = name.replace('+', '_') + desc = OpcodeDesc(self, name, index, methodname) + setattr(self.opcodedesc, name, desc) + self.opdescmap[index] = desc + # fill the ordered opdesc list + self.ordered_opdescs = lst = self.opdescmap.values() + lst.sort() + + def to_globals(self): + """NOT_RPYTHON. Add individual opcodes to the module constants.""" + g = globals() + g.update(self.opmap) + g['SLICE'] = self.opmap["SLICE+0"] + g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] + g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + + def __str__(self): + return "<%s bytecode>" % (self.name,) + + __repr__ = __str__ -class _baseopcodedesc: - pass -class opcodedesc(_baseopcodedesc): - """A namespace mapping OPCODE_NAME to OpcodeDescs.""" +# Initialization -class host_opcodedesc(_baseopcodedesc): - """A namespace mapping OPCODE_NAME to HostOpcodeDescs.""" +from pypy.rlib.unroll import unrolling_iterable + +from opcode import ( + opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) + +def load_pypy_opcode(): + import py + opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') + d = {} + execfile(str(opcode_path), d) + for name in __all__: + if name in d: + globals()[name] = d[name] + return d -for name, index in opmap.items(): - desc = OpcodeDesc(name, index) - setattr(opcodedesc, name, desc) - opdescmap[index] = desc - -for name, index in host_opmap.items(): - desc = HostOpcodeDesc(name, index) - setattr(host_opcodedesc, name, desc) - -lst = opdescmap.values() -lst.sort() -unrolling_opcode_descs = unrolling_iterable(lst) +load_pypy_opcode() +del load_pypy_opcode -del name, index, desc, lst +bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT) +host_bytecode_spec = BytecodeSpec('host', host_opmap, host_HAVE_ARGUMENT) +bytecode_spec.to_globals() + +opcode_method_names = bytecode_spec.method_names +opcodedesc = bytecode_spec.opcodedesc + +unrolling_all_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs + host_bytecode_spec.ordered_opdescs) +unrolling_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs) Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Tue Apr 6 23:35:35 2010 @@ -213,6 +213,10 @@ else: prof = None + import gc + a, b, c = gc.get_threshold() + gc.set_threshold(a * 2, b * 2, c * 8) + t = translator.TranslationContext(config=config) pdb_plus_show = PdbPlusShow(t) # need a translator to support extended commands From afa at codespeak.net Tue Apr 6 23:37:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Apr 2010 23:37:25 +0200 (CEST) Subject: [pypy-svn] r73472 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406213725.19DA6282B9C@codespeak.net> Author: afa Date: Tue Apr 6 23:37:23 2010 New Revision: 73472 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Check the type of 'self' when calling object_member.__get__. This cannot be done by the base class GetSetProperty, which performs a "typedef check" when we really want "isinstance". Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Tue Apr 6 23:37:23 2010 @@ -7,7 +7,7 @@ from pypy.rpython.annlowlevel import llhelper from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch from pypy.objspace.std.typeobject import W_TypeObject, _CPYTYPE, call__Type from pypy.objspace.std.typetype import _precheck_for_new from pypy.objspace.std.objectobject import W_ObjectObject @@ -40,9 +40,10 @@ class W_GetSetPropertyEx(GetSetProperty): - def __init__(self, getset): + def __init__(self, getset, pto): self.getset = getset self.name = rffi.charp2str(getset.c_name) + self.pto = pto doc = set = get = None if doc: doc = rffi.charp2str(getset.c_doc) @@ -55,12 +56,13 @@ tag="cpyext_1") def PyDescr_NewGetSet(space, getset, pto): - return space.wrap(W_GetSetPropertyEx(getset)) + return space.wrap(W_GetSetPropertyEx(getset, pto)) class W_MemberDescr(GetSetProperty): - def __init__(self, member): + def __init__(self, member, pto): self.member = member self.name = rffi.charp2str(member.c_name) + self.pto = pto flags = rffi.cast(lltype.Signed, member.c_flags) doc = set = None if member.c_doc: @@ -97,7 +99,7 @@ if not name: break name = rffi.charp2str(name) - w_descr = space.wrap(W_MemberDescr(member)) + w_descr = space.wrap(W_MemberDescr(member, pto)) dict_w[name] = w_descr i += 1 @@ -231,24 +233,34 @@ self.pyo = lltype.nullptr(PyObject.TO) # XXX handle borrowed objects here +def check_descr(space, w_self, pto): + w_type = from_ref(space, (rffi.cast(PyObject, pto))) + if not space.is_true(space.isinstance(w_self, w_type)): + raise DescrMismatch() + class GettersAndSetters: def getter(self, space, w_self): + check_descr(space, w_self, self.pto) return generic_cpy_call( space, self.getset.c_get, w_self, self.getset.c_closure) def setter(self, space, w_self, w_value): + check_descr(space, w_self, self.pto) return generic_cpy_call( space, self.getset.c_set, w_self, w_value, self.getset.c_closure) def member_getter(self, space, w_self): + check_descr(space, w_self, self.pto) return PyMember_GetOne(space, w_self, self.member) def member_delete(self, space, w_self): + check_descr(space, w_self, self.pto) PyMember_SetOne(space, w_self, self.member, None) def member_setter(self, space, w_self, w_value): + check_descr(space, w_self, self.pto) PyMember_SetOne(space, w_self, self.member, w_value) def c_type_descr__call__(space, w_type, __args__): From antoine at codespeak.net Tue Apr 6 23:42:12 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Tue, 6 Apr 2010 23:42:12 +0200 (CEST) Subject: [pypy-svn] r73473 - in pypy/branch/decouple-host-opcodes/pypy: interpreter translator/goal Message-ID: <20100406214212.8FF4E282B9C@codespeak.net> Author: antoine Date: Tue Apr 6 23:42:11 2010 New Revision: 73473 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Log: Remove some, well, experiments Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Tue Apr 6 23:42:11 2010 @@ -230,7 +230,7 @@ if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, next_instr, ec) - if 1 or we_are_translated(): + if we_are_translated(): from pypy.rlib import rstack # for resume points for opdesc in unrolling_all_opcode_descs: Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Tue Apr 6 23:42:11 2010 @@ -213,10 +213,6 @@ else: prof = None - import gc - a, b, c = gc.get_threshold() - gc.set_threshold(a * 2, b * 2, c * 8) - t = translator.TranslationContext(config=config) pdb_plus_show = PdbPlusShow(t) # need a translator to support extended commands From xoraxax at codespeak.net Wed Apr 7 00:41:43 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 00:41:43 +0200 (CEST) Subject: [pypy-svn] r73474 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406224143.D41E0282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 00:41:42 2010 New Revision: 73474 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Fix translation (hopefully). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 00:41:42 2010 @@ -26,6 +26,7 @@ # CPython 2.4 compatibility from py.builtin import BaseException from pypy.tool.sourcetools import func_with_new_name +from pypy.objspace.std.noneobject import W_NoneObject DEBUG_WRAPPER = False @@ -245,7 +246,7 @@ "List": "space.w_list", "Unicode": "space.w_unicode", 'Bool': 'space.w_bool', - 'None': 'space.w_None', + 'None': 'space.gettypeobject(W_NoneObject.typedef)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) @@ -419,6 +420,7 @@ object_pto.c_tp_bases = make_ref(space, space.newtuple([])) inherit_slots(space, type_pto, space.w_object) + #_____________________________________________________ # Build the bridge DLL, Allow extension DLLs to call # back into Pypy space functions From xoraxax at codespeak.net Wed Apr 7 00:47:46 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 00:47:46 +0200 (CEST) Subject: [pypy-svn] r73475 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406224746.93AC7282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 00:47:44 2010 New Revision: 73475 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix translation. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 00:47:44 2010 @@ -326,12 +326,9 @@ ref[0] = rffi.cast(rffi.INT, space.int_w(space.len(w_obj))) return 1 -def get_helper(space, func): - return llhelper(func.api_func.functype, func.api_func.get_wrapper(space)) - def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = get_helper(space, str_segcount) + c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, str_segcount.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) From xoraxax at codespeak.net Wed Apr 7 00:52:56 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 00:52:56 +0200 (CEST) Subject: [pypy-svn] r73476 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406225256.1187D282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 00:52:54 2010 New Revision: 73476 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Move if to appropriate HEAPTYPE branch. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 00:52:54 2010 @@ -328,7 +328,8 @@ def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, str_segcount.api_func.get_wrapper(space)) + c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, + str_segcount.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) @@ -361,10 +362,9 @@ base_pyo = rffi.cast(PyObject, obj_pto.c_tp_base) Py_DecRef(space, obj_pto.c_tp_bases) Py_DecRef(space, obj_pto.c_tp_cache) # lets do it like cpython - if obj_pto.c_tp_as_buffer: - lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') - # this does not have a refcount if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: + if obj_pto.c_tp_as_buffer: + lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') Py_DecRef(space, base_pyo) lltype.free(obj_pto.c_tp_name, flavor="raw") obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) From xoraxax at codespeak.net Wed Apr 7 00:58:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 00:58:05 +0200 (CEST) Subject: [pypy-svn] r73477 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406225805.D587A282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 00:58:04 2010 New Revision: 73477 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Simplify the nonetype fishing. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 00:58:04 2010 @@ -26,7 +26,6 @@ # CPython 2.4 compatibility from py.builtin import BaseException from pypy.tool.sourcetools import func_with_new_name -from pypy.objspace.std.noneobject import W_NoneObject DEBUG_WRAPPER = False @@ -246,7 +245,7 @@ "List": "space.w_list", "Unicode": "space.w_unicode", 'Bool': 'space.w_bool', - 'None': 'space.gettypeobject(W_NoneObject.typedef)', + 'None': 'space.type(space.w_None)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From xoraxax at codespeak.net Wed Apr 7 01:06:57 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 01:06:57 +0200 (CEST) Subject: [pypy-svn] r73478 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406230657.8C8BE282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 01:06:56 2010 New Revision: 73478 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Zero here to fix translation hopefully this time. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 01:06:56 2010 @@ -166,7 +166,7 @@ @specialize.memo() def get_new_method_def(space): from pypy.module.cpyext.modsupport import PyMethodDef - ptr = lltype.malloc(PyMethodDef, flavor="raw") + ptr = lltype.malloc(PyMethodDef, flavor="raw", zero=True) ptr.c_ml_name = rffi.str2charp("__new__") ptr.c_ml_meth = rffi.cast(PyCFunction, llhelper(tp_new_wrapper.api_func.functype, From fijal at codespeak.net Wed Apr 7 01:23:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 01:23:27 +0200 (CEST) Subject: [pypy-svn] r73479 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100406232327.50D5A282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 01:23:25 2010 New Revision: 73479 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Implement next part - str_getreadbuffer Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Wed Apr 7 01:23:25 2010 @@ -160,4 +160,7 @@ assert c_buf.c_bf_getsegcount(py_obj, ref) == 1 assert ref[0] == 10 lltype.free(ref, flavor='raw') + ref = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') + assert c_buf.c_bf_getreadbuffer(py_obj, 0, ref) + lltype.free(ref, flavor='raw') Py_DecRef(space, py_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 01:23:25 2010 @@ -319,6 +319,10 @@ # hopefully this does not clash with the memory model assumed in # extension modules + at specialize.memo() +def get_helper(space, func): + return llhelper(func.api_func.functype, func.api_func.get_wrapper(space)) + @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): @@ -326,10 +330,21 @@ ref[0] = rffi.cast(rffi.INT, space.int_w(space.len(w_obj))) return 1 + at cpython_api([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, + external=False, error=-1) +def str_getreadbuffer(space, w_str, segment, ref): + if segment != 0: + raise OperationError(space.w_SystemError, space.wrap + ("accessing non-existent string segment")) + pyref = make_ref(space, w_str, steal=True) + py_str = rffi.cast(PyStringObject, pyref) + ref[0] = py_str.c_buffer + return 1 + def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, - str_segcount.api_func.get_wrapper(space)) + c_buf.c_bf_getsegcount = get_helper(space, str_segcount) + c_buf.c_bf_getreadbuffer = get_helper(space, str_getreadbuffer) pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) From fijal at codespeak.net Wed Apr 7 01:32:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 01:32:14 +0200 (CEST) Subject: [pypy-svn] r73480 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406233214.37D04282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 01:32:12 2010 New Revision: 73480 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: fix translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 01:32:12 2010 @@ -319,10 +319,6 @@ # hopefully this does not clash with the memory model assumed in # extension modules - at specialize.memo() -def get_helper(space, func): - return llhelper(func.api_func.functype, func.api_func.get_wrapper(space)) - @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): @@ -343,8 +339,10 @@ def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - c_buf.c_bf_getsegcount = get_helper(space, str_segcount) - c_buf.c_bf_getreadbuffer = get_helper(space, str_getreadbuffer) + c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype, + str_segcount.api_func.get_wrapper(space)) + c_buf.c_bf_getreadbuffer = llhelper(str_getreadbuffer.api_func.functype, + str_getreadbuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) From xoraxax at codespeak.net Wed Apr 7 01:43:22 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 01:43:22 +0200 (CEST) Subject: [pypy-svn] r73481 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406234322.765E8282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 01:43:20 2010 New Revision: 73481 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Log: Fix annotation cleverness (it was annotated as void). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Wed Apr 7 01:43:20 2010 @@ -57,7 +57,7 @@ return space.int_w(space.len(w_obj)) @cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) -def PyDict_Next(space, p, ppos, pkey, pvalue): +def PyDict_Next(space, w_obj, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Py_ssize_t referred to by ppos must be initialized to 0 prior to the first call to this function to start the iteration; the @@ -98,6 +98,8 @@ } Py_DECREF(o); }""" + if w_obj is None: + return 0 raise NotImplementedError From xoraxax at codespeak.net Wed Apr 7 01:43:47 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 01:43:47 +0200 (CEST) Subject: [pypy-svn] r73482 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100406234347.16CD9282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 01:43:45 2010 New Revision: 73482 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Use llhelper in RPython code. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Wed Apr 7 01:43:45 2010 @@ -22,6 +22,8 @@ def startup(self, space): state = space.fromcache(State) + from pypy.module.cpyext.typeobject import setup_new_method_def + setup_new_method_def(space) if not we_are_translated(): space.setattr(space.wrap(self), space.wrap('api_lib'), Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 01:43:45 2010 @@ -168,13 +168,16 @@ from pypy.module.cpyext.modsupport import PyMethodDef ptr = lltype.malloc(PyMethodDef, flavor="raw", zero=True) ptr.c_ml_name = rffi.str2charp("__new__") - ptr.c_ml_meth = rffi.cast(PyCFunction, - llhelper(tp_new_wrapper.api_func.functype, - tp_new_wrapper.api_func.get_wrapper(space))) rffi.setintfield(ptr, 'c_ml_flags', METH_VARARGS | METH_KEYWORDS) ptr.c_ml_doc = rffi.str2charp("T.__new__(S, ...) -> a new object with type S, a subtype of T") return ptr +def setup_new_method_def(space): + ptr = get_new_method_def(space) + ptr.c_ml_meth = rffi.cast(PyCFunction, + llhelper(tp_new_wrapper.api_func.functype, + tp_new_wrapper.api_func.get_wrapper(space))) + def add_tp_new_wrapper(space, dict_w, pto): if "__new__" in dict_w: return From xoraxax at codespeak.net Wed Apr 7 01:54:33 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 01:54:33 +0200 (CEST) Subject: [pypy-svn] r73483 - pypy/branch/cpython-extension/pypy/module/cpyext/src Message-ID: <20100406235433.56112282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 01:54:31 2010 New Revision: 73483 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/varargwrapper.c Log: Rename function to PyTuple_Pack. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/varargwrapper.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/varargwrapper.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/varargwrapper.c Wed Apr 7 01:54:31 2010 @@ -1,7 +1,7 @@ #include #include -PyObject * PyPyTuple_Pack(Py_ssize_t size, ...) +PyObject * PyTuple_Pack(Py_ssize_t size, ...) { va_list ap; PyObject *cur, *tuple; From trundle at codespeak.net Wed Apr 7 01:55:29 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Wed, 7 Apr 2010 01:55:29 +0200 (CEST) Subject: [pypy-svn] r73484 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100406235529.3B964282B9C@codespeak.net> Author: trundle Date: Wed Apr 7 01:55:27 2010 New Revision: 73484 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_AS_UNICODE and PyUnicode_AsUnicode. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 01:55:27 2010 @@ -290,7 +290,7 @@ PyUnicodeObjectStruct = lltype.ForwardReference() PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) PyUnicodeObjectFields = (PyObjectFields + - (("buffer", rffi.VOIDP), ("size", Py_ssize_t))) + (("buffer", rffi.CWCHARP), ("size", Py_ssize_t))) cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) VA_TP_LIST = {} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/unicodeobject.h Wed Apr 7 01:55:27 2010 @@ -6,13 +6,6 @@ #endif -typedef struct { - PyObject_HEAD - void *buffer; - Py_ssize_t size; -} PyUnicodeObject; - - typedef unsigned int Py_UCS4; #ifdef HAVE_USABLE_WCHAR_T #define PY_UNICODE_TYPE wchar_t @@ -23,7 +16,14 @@ #endif typedef PY_UNICODE_TYPE Py_UNICODE; - + +typedef struct { + PyObject_HEAD + Py_UNICODE *buffer; + Py_ssize_t size; +} PyUnicodeObject; + + #ifdef __cplusplus } #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Wed Apr 7 01:55:27 2010 @@ -77,7 +77,7 @@ elif isinstance(w_obj, W_UnicodeObject): py_obj_unicode = lltype.malloc(PyUnicodeObject.TO, flavor='raw', zero=True) py_obj_unicode.c_size = len(space.unicode_w(w_obj)) - py_obj_unicode.c_buffer = lltype.nullptr(rffi.VOIDP.TO) + py_obj_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) pto = make_ref(space, space.w_unicode) py_obj = rffi.cast(PyObject, py_obj_unicode) py_obj.c_ob_refcnt = 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 7 01:55:27 2010 @@ -5653,12 +5653,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], {Py_UNICODE*}) -def PyUnicode_AS_UNICODE(space, o): - """Return a pointer to the internal Py_UNICODE buffer of the object. o - has to be a PyUnicodeObject (not checked).""" - raise NotImplementedError - @cpython_api([], rffi.INT_real) def PyUnicode_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. @@ -5726,12 +5720,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], {Py_UNICODE*}) -def PyUnicode_AsUnicode(space, unicode): - """Return a read-only pointer to the Unicode object's internal Py_UNICODE - buffer, NULL if unicode is not a Unicode object.""" - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyUnicode_GetSize(space, unicode): """Return the length of the Unicode object. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Wed Apr 7 01:55:27 2010 @@ -9,11 +9,17 @@ unichar = rffi.sizeof(Py_UNICODE) assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 4 * unichar - def test_AS_DATA(self, space, api): + def test_AS(self, space, api): word = space.wrap(u'spam') array = rffi.cast(rffi.CWCHARP, api.PyUnicode_AS_DATA(word)) + array2 = api.PyUnicode_AS_UNICODE(word) + array3 = api.PyUnicode_AsUnicode(word) for (i, char) in enumerate(space.unwrap(word)): assert array[i] == char + assert array2[i] == char + assert array3[i] == char + raises(TypeError, api.PyUnicode_AsUnicode(space.wrap('spam'))) + api.PyErr_Clear() def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 7 01:55:27 2010 @@ -1,3 +1,4 @@ +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, @@ -58,13 +59,7 @@ def PyUnicode_AS_DATA(space, ref): """Return a pointer to the internal buffer of the object. o has to be a PyUnicodeObject (not checked).""" - ref_unicode = rffi.cast(PyUnicodeObject, ref) - if not ref_unicode.c_buffer: - # Copy unicode buffer - w_unicode = from_ref(space, ref) - u = space.unicode_w(w_unicode) - ref_unicode.c_buffer = rffi.cast(rffi.VOIDP, rffi.unicode2wcharp(u)) - return rffi.cast(rffi.CCHARP, ref_unicode.c_buffer) + return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref)) @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyUnicode_GET_DATA_SIZE(space, w_obj): @@ -81,3 +76,24 @@ in your code for properly supporting 64-bit systems.""" assert isinstance(w_obj, unicodeobject.W_UnicodeObject) return space.int_w(space.len(w_obj)) + + at cpython_api([PyObject], rffi.CWCHARP, error=CANNOT_FAIL) +def PyUnicode_AS_UNICODE(space, ref): + """Return a pointer to the internal Py_UNICODE buffer of the object. ref + has to be a PyUnicodeObject (not checked).""" + ref_unicode = rffi.cast(PyUnicodeObject, ref) + if not ref_unicode.c_buffer: + # Copy unicode buffer + w_unicode = from_ref(space, ref) + u = space.unicode_w(w_unicode) + ref_unicode.c_buffer = rffi.unicode2wcharp(u) + return ref_unicode.c_buffer + + at cpython_api([PyObject], rffi.CWCHARP, error=0) +def PyUnicode_AsUnicode(space, ref): + """Return a read-only pointer to the Unicode object's internal Py_UNICODE + buffer, NULL if unicode is not a Unicode object.""" + if not PyUnicode_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected unicode object")) + return PyUnicode_AS_UNICODE(space, ref) From antoine at codespeak.net Wed Apr 7 01:57:00 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Wed, 7 Apr 2010 01:57:00 +0200 (CEST) Subject: [pypy-svn] r73485 - in pypy/branch/decouple-host-opcodes/pypy: interpreter module/__builtin__ objspace/std/test translator/goal Message-ID: <20100406235700.5C40A282B9C@codespeak.net> Author: antoine Date: Wed Apr 7 01:56:58 2010 New Revision: 73485 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py pypy/branch/decouple-host-opcodes/pypy/module/__builtin__/app_inspect.py pypy/branch/decouple-host-opcodes/pypy/objspace/std/test/test_obj.py pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Log: Implement some 2.7 opcodes (enough to pass objspace tests untranslated) Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py Wed Apr 7 01:56:58 2010 @@ -239,6 +239,8 @@ self.pushvalue(w_value) def peekvalue(self, index_from_top=0): + # NOTE: top of the stack is peekvalue(0). + # Contrast this with CPython where it's PEEK(-1). index_from_top = hint(index_from_top, promote=True) index = self.valuestackdepth + ~index_from_top assert index >= 0, "peek past the bottom of the stack" Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Wed Apr 7 01:56:58 2010 @@ -261,7 +261,22 @@ else: # when we are not translated, a list lookup is much faster methodname = self.opcode_method_names[opcode] - res = getattr(self, methodname)(oparg, next_instr) + try: + meth = getattr(self, methodname) + except AttributeError: + raise BytecodeCorruption("unimplemented opcode, ofs=%d, code=%d, name=%s" % + (self.last_instr, opcode, methodname)) + try: + res = meth(oparg, next_instr) + except Exception: + if 0: + import dis, sys + print "*** %s at offset %d (%s)" % (sys.exc_info()[0], self.last_instr, methodname) + try: + dis.dis(co_code) + except: + pass + raise if res is not None: next_instr = res @@ -997,7 +1012,7 @@ self.pushvalue(w_result) def MISSING_OPCODE(self, oparg, next_instr): - ofs = next_instr - 1 + ofs = self.last_instr c = self.pycode.co_code[ofs] name = self.pycode.co_name raise BytecodeCorruption("unknown opcode, ofs=%d, code=%d, name=%s" % @@ -1007,6 +1022,9 @@ class __extend__(pyframe.PyPyFrame): + """ + Execution of PyPy opcodes (mostly derived from Python 2.5.2). + """ bytecode_spec = bytecode_spec opcode_method_names = bytecode_spec.method_names opcodedesc = bytecode_spec.opcodedesc @@ -1025,6 +1043,9 @@ host_version_info = sys.version_info class __extend__(pyframe.HostPyFrame): + """ + Execution of host (CPython) opcodes. + """ bytecode_spec = host_bytecode_spec opcode_method_names = host_bytecode_spec.method_names opcodedesc = host_bytecode_spec.opcodedesc @@ -1051,6 +1072,40 @@ else: raise BytecodeCorruption + def POP_JUMP_IF_FALSE(self, jumpto, next_instr): + w_cond = self.popvalue() + if not self.space.is_true(w_cond): + next_instr = jumpto + return next_instr + + def POP_JUMP_IF_TRUE(self, jumpto, next_instr): + w_cond = self.popvalue() + if self.space.is_true(w_cond): + return jumpto + return next_instr + + def JUMP_IF_FALSE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if not self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def JUMP_IF_TRUE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def LIST_APPEND(self, oparg, next_instr): + w = self.popvalue() + if host_version_info < (2, 7): + v = self.popvalue() + else: + v = self.peekvalue(oparg - 1) + self.space.call_method(v, 'append', w) + ### ____________________________________________________________ ### Modified: pypy/branch/decouple-host-opcodes/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/module/__builtin__/app_inspect.py Wed Apr 7 01:56:58 2010 @@ -64,9 +64,7 @@ if isinstance(obj, types.ModuleType): try: - result = obj.__dict__.keys() - if not isinstance(result, list): - raise TypeError("expected __dict__.keys() to be a list") + result = list(obj.__dict__.keys()) result.sort() return result except AttributeError: Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/test/test_obj.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/test/test_obj.py Wed Apr 7 01:56:58 2010 @@ -7,11 +7,14 @@ cpython_behavior = (not option.runappdirect or not hasattr(sys, 'pypy_translation_info')) - cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_version = cls.space.wrap(tuple(sys.version_info)) def test_hash_builtin(self): if not self.cpython_behavior: skip("on pypy-c id == hash is not guaranteed") + if self.cpython_version >= (2, 7): + skip("on CPython >= 2.7, id != hash") import sys o = object() assert (hash(o) & sys.maxint) == (id(o) & sys.maxint) @@ -33,7 +36,7 @@ class X(object): pass x = X() - if self.cpython_behavior: + if self.cpython_behavior and self.cpython_version < (2, 7): assert (hash(x) & sys.maxint) == (id(x) & sys.maxint) assert hash(x) == object.__hash__(x) Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/translate.py Wed Apr 7 01:56:58 2010 @@ -213,6 +213,14 @@ else: prof = None + import gc + try: + # Reduce CPython GC pressure by raising the thresholds + a, b, c = gc.get_threshold() + gc.set_threshold(a * 2, b * 3, c * 15) + except (AttributeError, NotImplementedError): + pass + t = translator.TranslationContext(config=config) pdb_plus_show = PdbPlusShow(t) # need a translator to support extended commands From antoine at codespeak.net Wed Apr 7 02:00:57 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Wed, 7 Apr 2010 02:00:57 +0200 (CEST) Subject: [pypy-svn] r73486 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100407000057.42D37282B9C@codespeak.net> Author: antoine Date: Wed Apr 7 02:00:55 2010 New Revision: 73486 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Log: Add a note about missing 2.7 opcodes. Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Wed Apr 7 02:00:55 2010 @@ -1105,6 +1105,11 @@ else: v = self.peekvalue(oparg - 1) self.space.call_method(v, 'append', w) + + # XXX: Missing 2.7 opcodes: + # - BUILD_SET, SET_ADD: set literals and set comprehensions + # - MAP_ADD: dict comprehensions + # - SETUP_WITH: `with` statement ### ____________________________________________________________ ### From fijal at codespeak.net Wed Apr 7 02:05:41 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 02:05:41 +0200 (CEST) Subject: [pypy-svn] r73487 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407000541.98D73282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 02:05:40 2010 New Revision: 73487 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Fix PyObject_Size, test coming Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 7 02:05:40 2010 @@ -67,7 +67,7 @@ @cpython_api([PyObject], Py_ssize_t, error=-1) def PyObject_Size(space, w_obj): - return space.len(w_obj) + return space.int_w(space.len(w_obj)) @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallable_Check(space, w_obj): From fijal at codespeak.net Wed Apr 7 02:06:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 02:06:44 +0200 (CEST) Subject: [pypy-svn] r73488 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100407000644.5966D36C21D@codespeak.net> Author: fijal Date: Wed Apr 7 02:06:43 2010 New Revision: 73488 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Add a test for PyObject_Size Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 7 02:06:43 2010 @@ -61,3 +61,6 @@ w_d = space.newdict() space.setitem(w_d, space.wrap("a key!"), space.wrap(72)) assert space.unwrap(api.PyObject_GetItem(w_d, space.wrap("a key!"))) == 72 + + def test_size(self, space, api): + assert api.PyObject_Size(space.newlist([space.w_None])) == 1 From antoine at codespeak.net Wed Apr 7 02:15:33 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Wed, 7 Apr 2010 02:15:33 +0200 (CEST) Subject: [pypy-svn] r73489 - pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/test Message-ID: <20100407001533.471BA282B9C@codespeak.net> Author: antoine Date: Wed Apr 7 02:15:31 2010 New Revision: 73489 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/test/test_compiler.py Log: More test fixes for 2.7 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/test/test_compiler.py Wed Apr 7 02:15:31 2010 @@ -22,11 +22,15 @@ """ def run(self, source): + import sys source = str(py.code.Source(source)) space = self.space code = compile_with_astcompiler(source, 'exec', space) - print - code.dump() + # 2.7 bytecode is too different, the standard `dis` module crashes + # when trying to display pypy (2.5-like) bytecode. + if sys.version_info < (2, 7): + print + code.dump() w_dict = space.newdict() code.exec_code(space, w_dict, w_dict) return w_dict @@ -39,7 +43,12 @@ pyco_expr = PyCode._from_code(space, co_expr) w_res = pyco_expr.exec_code(space, w_dict, w_dict) res = space.str_w(space.repr(w_res)) - assert res == repr(expected) + if not isinstance(expected, float): + assert res == repr(expected) + else: + # Float representation can vary a bit between interpreter + # versions, compare the numbers instead. + assert eval(res) == expected def simple_test(self, source, evalexpr, expected): w_g = self.run(source) From xoraxax at codespeak.net Wed Apr 7 02:22:12 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 02:22:12 +0200 (CEST) Subject: [pypy-svn] r73490 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407002212.E47D9282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 02:22:11 2010 New Revision: 73490 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Generate correct decl code for the standalone case. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 02:22:11 2010 @@ -522,7 +522,7 @@ pypy_macros_h = udir.join('pypy_macros.h') pypy_macros_h.write('\n'.join(pypy_macros)) -def generate_decls_and_callbacks(db, export_symbols, api_struct=True): +def generate_decls_and_callbacks(db, export_symbols, api_struct=True, globals_are_pointers=True): # implement function callbacks and generate function decls functions = [] pypy_decls = [] @@ -543,6 +543,8 @@ functions.append('%s\n%s\n' % (header, body)) pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, (typ, expr) in GLOBALS.iteritems(): + if not globals_are_pointers: + typ = typ.replace("*", "") pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) for name in VA_TP_LIST: name_no_star = process_va_name(name) @@ -598,7 +600,7 @@ generate_macros(export_symbols, rename, False) - functions = generate_decls_and_callbacks(db, [], api_struct=False) + functions = generate_decls_and_callbacks(db, [], api_struct=False, globals_are_pointers=False) code = "#include \n" + "\n".join(functions) eci = build_eci(False, export_symbols, code) From xoraxax at codespeak.net Wed Apr 7 02:34:52 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 02:34:52 +0200 (CEST) Subject: [pypy-svn] r73491 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407003452.558BF282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 02:34:50 2010 New Revision: 73491 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix SyntaxError while translation. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 02:34:50 2010 @@ -161,7 +161,7 @@ args_w = space.listview(w_args)[:] args_w.insert(0, w_self) w_args_new = space.newlist(args_w) - return space.call(space.lookup(space.w_type, space.wrap("__new__")), w_args_new, w_kwds) + return space.call(space.lookup(space.w_type, "__new__"), w_args_new, w_kwds) @specialize.memo() def get_new_method_def(space): From antoine at codespeak.net Wed Apr 7 02:47:07 2010 From: antoine at codespeak.net (antoine at codespeak.net) Date: Wed, 7 Apr 2010 02:47:07 +0200 (CEST) Subject: [pypy-svn] r73492 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100407004707.175C9282B9C@codespeak.net> Author: antoine Date: Wed Apr 7 02:47:06 2010 New Revision: 73492 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Log: Translating with 2.7? Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Wed Apr 7 02:47:06 2010 @@ -1040,7 +1040,8 @@ def STORE_MAP(self, zero, next_instr): raise BytecodeCorruption -host_version_info = sys.version_info +# Need to cast 2.7's "named struct" to a plain struct +host_version_info = tuple(sys.version_info) class __extend__(pyframe.HostPyFrame): """ @@ -1106,10 +1107,25 @@ v = self.peekvalue(oparg - 1) self.space.call_method(v, 'append', w) - # XXX: Missing 2.7 opcodes: - # - BUILD_SET, SET_ADD: set literals and set comprehensions - # - MAP_ADD: dict comprehensions - # - SETUP_WITH: `with` statement + # XXX Unimplemented 2.7 opcodes ---------------- + + # Set literals, set comprehensions + + def BUILD_SET(self, oparg, next_instr): + raise NotImplementedError("BUILD_SET") + + def SET_ADD(self, oparg, next_instr): + raise NotImplementedError("SET_ADD") + + # Dict comprehensions + + def MAP_ADD(self, oparg, next_instr): + raise NotImplementedError("MAP_ADD") + + # `with` statement + + def SETUP_WITH(self, oparg, next_instr): + raise NotImplementedError("SETUP_WITH") ### ____________________________________________________________ ### From xoraxax at codespeak.net Wed Apr 7 03:02:24 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 03:02:24 +0200 (CEST) Subject: [pypy-svn] r73493 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100407010224.A873D282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 03:02:23 2010 New Revision: 73493 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/boolobject.h pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Generate correct decl code, use struct hack with defines like cpython for py_none/py_false/py_true. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 03:02:23 2010 @@ -230,9 +230,9 @@ ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur - 'Py_None': ('PyObject*', 'space.w_None'), - 'Py_True': ('PyObject*', 'space.w_True'), - 'Py_False': ('PyObject*', 'space.w_False'), + '_Py_NoneStruct#': ('PyObject*', 'space.w_None'), + '_Py_TrueStruct#': ('PyObject*', 'space.w_True'), + '_Py_ZeroStruct#': ('PyObject*', 'space.w_False'), } for exc_name in exceptions.Module.interpleveldefs.keys(): @@ -251,7 +251,7 @@ def get_structtype_for_ctype(ctype): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr - return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] + return {"PyObject": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] PyTypeObject = lltype.ForwardReference() PyTypeObjectPtr = lltype.Ptr(PyTypeObject) @@ -543,7 +543,7 @@ functions.append('%s\n%s\n' % (header, body)) pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, (typ, expr) in GLOBALS.iteritems(): - if not globals_are_pointers: + if not globals_are_pointers and "#" in name: typ = typ.replace("*", "") pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) for name in VA_TP_LIST: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/boolobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/boolobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/boolobject.h Wed Apr 7 03:02:23 2010 @@ -7,6 +7,9 @@ extern "C" { #endif +#define Py_False ((PyObject *) &_Py_ZeroStruct) +#define Py_True ((PyObject *) &_Py_TrueStruct) + /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Wed Apr 7 03:02:23 2010 @@ -46,6 +46,7 @@ #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define Py_None (&_Py_NoneStruct) struct _typeobject; typedef void (*freefunc)(void *); From xoraxax at codespeak.net Wed Apr 7 03:05:19 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 03:05:19 +0200 (CEST) Subject: [pypy-svn] r73494 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407010519.DBAE0282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 03:05:18 2010 New Revision: 73494 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Oops. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 03:05:18 2010 @@ -251,7 +251,7 @@ def get_structtype_for_ctype(ctype): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr - return {"PyObject": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] + return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr}[ctype] PyTypeObject = lltype.ForwardReference() PyTypeObjectPtr = lltype.Ptr(PyTypeObject) From trundle at codespeak.net Wed Apr 7 03:47:31 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Wed, 7 Apr 2010 03:47:31 +0200 (CEST) Subject: [pypy-svn] r73495 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407014731.DC3C9282B9C@codespeak.net> Author: trundle Date: Wed Apr 7 03:47:30 2010 New Revision: 73495 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_tupleobject.py pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py Log: Add PyTuple_Size. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 7 03:47:30 2010 @@ -5519,14 +5519,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyTuple_Size(space, p): - """Take a pointer to a tuple object, and return the size of that tuple. - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) def PyTuple_GET_ITEM(space, p, pos): """Like PyTuple_GetItem(), but does no checking of its arguments. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_tupleobject.py Wed Apr 7 03:47:30 2010 @@ -6,4 +6,8 @@ def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 + atuple = space.newtuple([0, 1, 'yay']) + assert api.PyTuple_Size(atuple) == 3 + assert api.PyTuple_GET_SIZE(atuple) == 3 + raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/tupleobject.py Wed Apr 7 03:47:30 2010 @@ -1,3 +1,4 @@ +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) @@ -38,3 +39,10 @@ assert isinstance(w_t, W_TupleObject) return len(w_t.wrappeditems) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyTuple_Size(space, ref): + """Take a pointer to a tuple object, and return the size of that tuple.""" + if not PyTuple_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected tuple object")) + return PyTuple_GET_SIZE(space, ref) From xoraxax at codespeak.net Wed Apr 7 04:05:24 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 04:05:24 +0200 (CEST) Subject: [pypy-svn] r73496 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100407020524.6C4D3282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 04:05:22 2010 New Revision: 73496 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Failing skipped sre test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Wed Apr 7 04:05:22 2010 @@ -90,10 +90,15 @@ assert sre_compile.MAGIC == module.MAGIC import re import time - s = u"Foo " * 1000 + u"Bar" - prog = re.compile(ur"Foo.*Bar") - assert prog.match(s) - m = re.search(u"xyz", u"xyzxyz") - assert m - re._cache.clear() - re._cache_repl.clear() + try: + s = u"Foo " * 1000 + u"Bar" + prog = re.compile(ur"Foo.*Bar") + assert prog.match(s) + m = re.search(u"xyz", u"xyzxyz") + assert m + skip("Crashing in fijals code") + m = re.search("xyz", "xyzxyz") + assert m + finally: + re._cache.clear() + re._cache_repl.clear() From xoraxax at codespeak.net Wed Apr 7 04:32:19 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 04:32:19 +0200 (CEST) Subject: [pypy-svn] r73497 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407023219.A5FB9282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 04:32:18 2010 New Revision: 73497 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Add a test for re.search and fix str_getreadbuffer. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Wed Apr 7 04:32:18 2010 @@ -90,15 +90,12 @@ assert sre_compile.MAGIC == module.MAGIC import re import time - try: - s = u"Foo " * 1000 + u"Bar" - prog = re.compile(ur"Foo.*Bar") - assert prog.match(s) - m = re.search(u"xyz", u"xyzxyz") - assert m - skip("Crashing in fijals code") - m = re.search("xyz", "xyzxyz") - assert m - finally: - re._cache.clear() - re._cache_repl.clear() + s = u"Foo " * 1000 + u"Bar" + prog = re.compile(ur"Foo.*Bar") + assert prog.match(s) + m = re.search(u"xyz", u"xyzxyz") + assert m + m = re.search("xyz", "xyzxyz") + assert m + re._cache.clear() + re._cache_repl.clear() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 04:32:18 2010 @@ -332,13 +332,13 @@ @cpython_api([PyObject, lltype.Signed, rffi.VOIDPP], lltype.Signed, external=False, error=-1) def str_getreadbuffer(space, w_str, segment, ref): + from pypy.module.cpyext.stringobject import PyString_AsString if segment != 0: raise OperationError(space.w_SystemError, space.wrap ("accessing non-existent string segment")) pyref = make_ref(space, w_str, steal=True) - py_str = rffi.cast(PyStringObject, pyref) - ref[0] = py_str.c_buffer - return 1 + ref[0] = PyString_AsString(space, pyref) + return 1 # XXX is this correct? def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) From fijal at codespeak.net Wed Apr 7 04:43:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 04:43:33 +0200 (CEST) Subject: [pypy-svn] r73498 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407024333.237D9282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 04:43:31 2010 New Revision: 73498 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix return value Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Wed Apr 7 04:43:31 2010 @@ -161,6 +161,6 @@ assert ref[0] == 10 lltype.free(ref, flavor='raw') ref = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') - assert c_buf.c_bf_getreadbuffer(py_obj, 0, ref) + assert c_buf.c_bf_getreadbuffer(py_obj, 0, ref) == 10 lltype.free(ref, flavor='raw') Py_DecRef(space, py_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 7 04:43:31 2010 @@ -338,7 +338,7 @@ ("accessing non-existent string segment")) pyref = make_ref(space, w_str, steal=True) ref[0] = PyString_AsString(space, pyref) - return 1 # XXX is this correct? + return space.int_w(space.len(w_str)) def setup_string_buffer_procs(space, pto): c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) From fijal at codespeak.net Wed Apr 7 05:49:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 05:49:28 +0200 (CEST) Subject: [pypy-svn] r73499 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100407034928.3F58C282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 05:49:25 2010 New Revision: 73499 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Log: remove pointless assignment Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Wed Apr 7 05:49:25 2010 @@ -163,7 +163,6 @@ @jit.unroll_safe def dispatch_bytecode(self, co_code, next_instr, ec): space = self.space - cls = self.__class__ # PyFrame is really an abstract class assert self.__class__ is not pyframe.PyFrame while True: From micke at codespeak.net Wed Apr 7 12:45:55 2010 From: micke at codespeak.net (micke at codespeak.net) Date: Wed, 7 Apr 2010 12:45:55 +0200 (CEST) Subject: [pypy-svn] r73500 - pypy/build/testrunner Message-ID: <20100407104555.59D41282B9C@codespeak.net> Author: micke Date: Wed Apr 7 12:45:51 2010 New Revision: 73500 Modified: pypy/build/testrunner/runner.py Log: no Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Wed Apr 7 12:45:51 2010 @@ -362,7 +362,6 @@ if py.path.local(config_py_file).check(file=1): print >>out, "using config", config_py_file execfile(config_py_file, run_param.__dict__) - run_param.__dict__.pop('__builtins__', None) if run_param.cherrypick: for p in run_param.cherrypick: From xoraxax at codespeak.net Wed Apr 7 13:53:31 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 13:53:31 +0200 (CEST) Subject: [pypy-svn] r73504 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407115331.D1447282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 13:53:30 2010 New Revision: 73504 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Update TODO file. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Wed Apr 7 13:53:30 2010 @@ -4,21 +4,15 @@ - Those tasks are necessary to be able to call slots from C code correctly. - Additionally, implement further slot wrappers. This is necessary to call slots of PTOs defined in C. + - Complete the Members support. - Use a WeakKeyDictionary to count how often a PyObject is allocated for a given wrapped object and use this to assess whether optimizations are useful - - lltype.free in PyObject_Del() sometimes raise an exception (but all - tests pass). FIXED? - - Reported by Amaury - - Alexander cannot reproduce it - - replace @cpython_api(external=False) by another explicit name: all it does is a lltype function pointer, no C code involved. - - Implement Members. - - Fix distutil's build_ext to work with cpyext. - Extend the import machinery to load .so/.pyd files. @@ -26,4 +20,4 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. - - ob_type should be a PyTypeObject. + - Test cpyext together with the jit. From trundle at codespeak.net Wed Apr 7 14:17:08 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Wed, 7 Apr 2010 14:17:08 +0200 (CEST) Subject: [pypy-svn] r73505 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407121708.0DFCA282B9C@codespeak.net> Author: trundle Date: Wed Apr 7 14:17:07 2010 New Revision: 73505 Added: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Add PySys_GetObject. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Wed Apr 7 14:17:07 2010 @@ -53,6 +53,7 @@ import pypy.module.cpyext.iterator import pypy.module.cpyext.unicodeobject import pypy.module.cpyext.pycobject +import pypy.module.cpyext.sysmodule # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 7 14:17:07 2010 @@ -5409,12 +5409,6 @@ alias for void (*)(int).""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject, borrowed=True) -def PySys_GetObject(space, name): - """Return the object name from the sys module or NULL if it does - not exist, without setting an exception.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, {FILE*}], {FILE*}) def PySys_GetFile(space, name, def): """Return the FILE* associated with the object name in the Added: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py Wed Apr 7 14:17:07 2010 @@ -0,0 +1,17 @@ +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import rffi +from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api +from pypy.module.cpyext.pyobject import PyObject + + at cpython_api([rffi.CCHARP], PyObject, borrowed=True, error=CANNOT_FAIL) +def PySys_GetObject(space, name): + """Return the object name from the sys module or NULL if it does + not exist, without setting an exception.""" + w_name = rffi.charp2str(name) + try: + w_obj = space.sys.get(w_name) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + w_obj = None + return w_obj Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Wed Apr 7 14:17:07 2010 @@ -0,0 +1,14 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.rpython.lltypesystem import rffi + +class TestSysModule(BaseApiTest): + def test_sysmodule(self, space, api): + version_info = rffi.str2charp("version_info") + assert api.PySys_GetObject(version_info) + assert not api.PyErr_Occurred() + rffi.free_charp(version_info) + + i_do_not_exist = rffi.str2charp("i_do_not_exist") + assert not api.PySys_GetObject(i_do_not_exist) + assert not api.PyErr_Occurred() + rffi.free_charp(i_do_not_exist) From xoraxax at codespeak.net Wed Apr 7 16:08:28 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 16:08:28 +0200 (CEST) Subject: [pypy-svn] r73506 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407140828.77F80282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 16:08:23 2010 New Revision: 73506 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Change decl generation again to let it work on 32 bit systems where long and int collide as well as generally when the exceptions were broken. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 16:08:23 2010 @@ -526,6 +526,7 @@ # implement function callbacks and generate function decls functions = [] pypy_decls = [] + pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, func in sorted(FUNCTIONS.iteritems()): restype = db.gettype(func.restype).replace('@', '') args = [] @@ -541,11 +542,13 @@ for i in range(len(func.argtypes))) body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) functions.append('%s\n%s\n' % (header, body)) - pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, (typ, expr) in GLOBALS.iteritems(): - if not globals_are_pointers and "#" in name: + name_clean = name.replace("#", "") + if not globals_are_pointers: typ = typ.replace("*", "") - pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name.replace("#", ""))) + pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) + if not globals_are_pointers and "#" not in name: + pypy_decls.append("#define %s &%s" % (name, name,)) for name in VA_TP_LIST: name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Wed Apr 7 16:08:23 2010 @@ -97,5 +97,6 @@ assert m m = re.search("xyz", "xyzxyz") assert m + assert dir(m) re._cache.clear() re._cache_repl.clear() From xoraxax at codespeak.net Wed Apr 7 16:09:00 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 16:09:00 +0200 (CEST) Subject: [pypy-svn] r73507 - in pypy/branch/cpython-extension/pypy/translator: . test Message-ID: <20100407140900.24DDD282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 16:08:58 2010 New Revision: 73507 Modified: pypy/branch/cpython-extension/pypy/translator/driver.py pypy/branch/cpython-extension/pypy/translator/test/test_unsimplify.py pypy/branch/cpython-extension/pypy/translator/unsimplify.py Log: Fix call_final_function to handle cases where the entry_point is not the first graph annotated. Modified: pypy/branch/cpython-extension/pypy/translator/driver.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/driver.py (original) +++ pypy/branch/cpython-extension/pypy/translator/driver.py Wed Apr 7 16:08:58 2010 @@ -324,6 +324,7 @@ if self.entry_point: s = annotator.build_types(self.entry_point, self.inputtypes) + translator.entry_point_graph = annotator.bookkeeper.getdesc(self.entry_point).getuniquegraph() else: s = None Modified: pypy/branch/cpython-extension/pypy/translator/test/test_unsimplify.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/test/test_unsimplify.py (original) +++ pypy/branch/cpython-extension/pypy/translator/test/test_unsimplify.py Wed Apr 7 16:08:58 2010 @@ -9,6 +9,7 @@ def translate(func, argtypes, type_system="lltype"): t = TranslationContext() t.buildannotator().build_types(func, argtypes) + t.entry_point_graph = graphof(t, func) t.buildrtyper(type_system=type_system).specialize() return graphof(t, func), t Modified: pypy/branch/cpython-extension/pypy/translator/unsimplify.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/unsimplify.py (original) +++ pypy/branch/cpython-extension/pypy/translator/unsimplify.py Wed Apr 7 16:08:58 2010 @@ -168,7 +168,7 @@ if own_annhelper: annhelper.finish() - entry_point = translator.graphs[0] + entry_point = translator.entry_point_graph v = copyvar(translator.annotator, entry_point.getreturnvar()) extrablock = Block([v]) v_none = varoftype(lltype.Void) From afa at codespeak.net Wed Apr 7 16:27:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Apr 2010 16:27:51 +0200 (CEST) Subject: [pypy-svn] r73508 - in pypy/branch/cpython-extension/pypy: config doc/config translator translator/c translator/c/src translator/c/test Message-ID: <20100407142751.80111282B9C@codespeak.net> Author: afa Date: Wed Apr 7 16:27:49 2010 New Revision: 73508 Added: pypy/branch/cpython-extension/pypy/doc/config/translation.shared.txt (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/config/pypyoption.py pypy/branch/cpython-extension/pypy/config/translationoption.py pypy/branch/cpython-extension/pypy/translator/c/genc.py pypy/branch/cpython-extension/pypy/translator/c/src/main.h pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py pypy/branch/cpython-extension/pypy/translator/driver.py Log: Add the --shared option to translation. This allows to build pypy as a shared library, which can thus export the C API provided by cpyext. On windows, this builds two files: - pypy-c.exe.dll is the shared library containing all pypy - pypy-c.exe is a tiny executable that calls the main function in the dll. Modified: pypy/branch/cpython-extension/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/cpython-extension/pypy/config/pypyoption.py (original) +++ pypy/branch/cpython-extension/pypy/config/pypyoption.py Wed Apr 7 16:27:49 2010 @@ -62,12 +62,14 @@ module_dependencies = {} -module_suggests = { # the reason you want _rawffi is for ctypes, which - # itself needs the interp-level struct module - # because 'P' is missing from the app-level one - '_rawffi': [("objspace.usemodules.struct", True)], - 'cpyext': [("translation.secondaryentrypoints", "cpyext")], - } +module_suggests = { + # the reason you want _rawffi is for ctypes, which + # itself needs the interp-level struct module + # because 'P' is missing from the app-level one + "_rawffi": [("objspace.usemodules.struct", True)], + "cpyext": [("translation.secondaryentrypoints", "cpyext"), + ("translation.shared", sys.platform == "win32")], + } module_import_dependencies = { # no _rawffi if importing pypy.rlib.libffi raises ImportError Modified: pypy/branch/cpython-extension/pypy/config/translationoption.py ============================================================================== --- pypy/branch/cpython-extension/pypy/config/translationoption.py (original) +++ pypy/branch/cpython-extension/pypy/config/translationoption.py Wed Apr 7 16:27:49 2010 @@ -42,6 +42,9 @@ }, cmdline="-b --backend"), + BoolOption("shared", "Build as a shared library", + default=False, cmdline="--shared"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", default=True, cmdline="--log"), Added: pypy/branch/cpython-extension/pypy/doc/config/translation.shared.txt ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/doc/config/translation.shared.txt Wed Apr 7 16:27:49 2010 @@ -0,0 +1,2 @@ +Build pypy as a shared library or a DLL, with a small executable to run it. +This is necessary on Windows to expose the C API provided by the cpyext module. Modified: pypy/branch/cpython-extension/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/genc.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/genc.py Wed Apr 7 16:27:49 2010 @@ -78,14 +78,15 @@ self.outputfilename = outputfilename self.profbased = profbased - def _build(self, eci=ExternalCompilationInfo()): + def _build(self, eci=ExternalCompilationInfo(), shared=False): return self.platform.compile(self.cfiles, self.eci.merge(eci), - outputfilename=self.outputfilename) + outputfilename=self.outputfilename, + standalone=not shared) - def build(self): + def build(self, shared=False): if self.profbased: return self._do_profbased() - return self._build() + return self._build(shared=shared) def _do_profbased(self): ProfDriver, args = self.profbased @@ -252,6 +253,10 @@ if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 + if self.config.translation.shared: + defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup" + self.eci = self.eci.merge(ExternalCompilationInfo( + export_symbols=["pypy_main_startup"])) self.eci, cfile, extra = gen_source_standalone(db, modulename, targetdir, self.eci, @@ -422,6 +427,7 @@ class CStandaloneBuilder(CBuilder): standalone = True executable_name = None + shared_library_name = None def getprofbased(self): profbased = None @@ -462,9 +468,33 @@ return res.out, res.err return res.out - def compile(self): + def build_main_for_shared(self, shared_library_name, entrypoint): + self.shared_library_name = shared_library_name + # build main program + eci = self.get_eci() + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=[''' + int %s(argc, argv); + + int main(int argc, char* argv[]) + { %s(argc, argv); } + ''' % (entrypoint, entrypoint) + ], + libraries=[self.shared_library_name.new(ext='')] + )) + eci = eci.convert_sources_to_files( + cache_dir=self.targetdir) + outfilename = self.shared_library_name.new(ext='') + return self.translator.platform.compile( + [], eci, + outputfilename=str(outfilename)) + + def compile(self, exe_name=None): assert self.c_source_filename assert not self._compiled + + shared = self.config.translation.shared + if (self.config.translation.gcrootfinder == "asmgcc" or self.config.translation.force_make): extra_opts = [] @@ -475,16 +505,21 @@ else: compiler = CCompilerDriver(self.translator.platform, [self.c_source_filename] + self.extrafiles, - self.eci, profbased=self.getprofbased()) - self.executable_name = compiler.build() + self.eci, profbased=self.getprofbased(), + outputfilename=exe_name) + self.executable_name = compiler.build(shared=shared) + if shared: + self.executable_name = self.build_main_for_shared( + self.executable_name, "pypy_main_startup") assert self.executable_name self._compiled = True return self.executable_name - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): cfiles = [self.c_source_filename] + self.extrafiles mk = self.translator.platform.gen_makefile(cfiles, self.eci, - path=targetdir) + path=targetdir, + exe_name=exe_name) if self.has_profopt(): profopt = self.config.translation.profopt mk.definition('ABS_TARGET', '$(shell python -c "import sys,os; print os.path.abspath(sys.argv[1])" $(TARGET))') Modified: pypy/branch/cpython-extension/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/main.h (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/main.h Wed Apr 7 16:27:49 2010 @@ -15,11 +15,15 @@ #ifndef PYPY_NOT_MAIN_FILE +#ifndef PYPY_MAIN_FUNCTION +#define PYPY_MAIN_FUNCTION main +#endif + #ifdef MS_WINDOWS #include "src/winstuff.c" #endif -int main(int argc, char *argv[]) +int PYPY_MAIN_FUNCTION(int argc, char *argv[]) { char *errmsg; int i, exitcode; Modified: pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py Wed Apr 7 16:27:49 2010 @@ -16,11 +16,13 @@ class StandaloneTests(object): config = None - def compile(self, entry_point, debug=True): + def compile(self, entry_point, debug=True, shared=False): t = TranslationContext(self.config) t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() + t.config.translation.shared = shared + cbuilder = CStandaloneBuilder(t, entry_point, t.config) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) @@ -587,6 +589,17 @@ # The traceback stops at f() because it's the first function that # captures the AssertionError, which makes the program abort. + def test_shared(self): + def f(argv): + print len(argv) + def entry_point(argv): + f(argv) + return 0 + t, cbuilder = self.compile(entry_point, shared=True) + assert cbuilder.shared_library_name is not None + assert cbuilder.shared_library_name != cbuilder.executable_name + out, err = cbuilder.cmdexec("a b") + assert out == "3" class TestMaemo(TestStandalone): def setup_class(cls): Modified: pypy/branch/cpython-extension/pypy/translator/driver.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/driver.py (original) +++ pypy/branch/cpython-extension/pypy/translator/driver.py Wed Apr 7 16:27:49 2010 @@ -516,12 +516,16 @@ exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() shutil.copy(str(exename), str(newexename)) + if self.cbuilder.shared_library_name is not None: + soname = self.cbuilder.shared_library_name + newsoname = newexename.new(basename=soname.basename) + shutil.copy(str(soname), str(newsoname)) self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) def task_compile_c(self): # xxx messy cbuilder = self.cbuilder - cbuilder.compile() + cbuilder.compile(exe_name=self.compute_exe_name().basename) if self.standalone: self.c_entryp = cbuilder.executable_name From afa at codespeak.net Wed Apr 7 18:24:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Apr 2010 18:24:29 +0200 (CEST) Subject: [pypy-svn] r73509 - pypy/branch/cpython-extension/pypy/translator/c/src Message-ID: <20100407162429.12356282B9C@codespeak.net> Author: afa Date: Wed Apr 7 18:24:26 2010 New Revision: 73509 Modified: pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h Log: Fix for my gcc-4.1.2 on debian which doesnt' define LLONG_MAX by default Modified: pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h Wed Apr 7 18:24:26 2010 @@ -15,6 +15,13 @@ #include +#ifndef LLONG_MAX +#define LLONG_MAX __LONG_LONG_MAX__ +#endif +#ifndef LLONG_MIN +#define LLONG_MIN (-LLONG_MAX - 1LL) +#endif + #if INT_MAX != 2147483647 # error "unsupported value for INT_MAX" #endif From trundle at codespeak.net Wed Apr 7 18:25:17 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Wed, 7 Apr 2010 18:25:17 +0200 (CEST) Subject: [pypy-svn] r73511 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407162517.E8D2C282B9C@codespeak.net> Author: trundle Date: Wed Apr 7 18:25:15 2010 New Revision: 73511 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Log: Fix borrowed reference handling in PySys_GetObject and write the tests in C. The tests fail though because the test runner can't really handle that the borrowed references still live on shutdown. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py Wed Apr 7 18:25:15 2010 @@ -1,17 +1,14 @@ from pypy.interpreter.error import OperationError -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api -from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.pyobject import PyObject, register_container @cpython_api([rffi.CCHARP], PyObject, borrowed=True, error=CANNOT_FAIL) def PySys_GetObject(space, name): """Return the object name from the sys module or NULL if it does not exist, without setting an exception.""" w_name = rffi.charp2str(name) - try: - w_obj = space.sys.get(w_name) - except OperationError, e: - if not e.match(space, space.w_AttributeError): - raise - w_obj = None + w_dict = space.sys.getdict() + w_obj = space.finditem_str(w_dict, w_name) + register_container(space, w_dict) return w_obj Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Wed Apr 7 18:25:15 2010 @@ -1,14 +1,15 @@ -from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi -class TestSysModule(BaseApiTest): - def test_sysmodule(self, space, api): - version_info = rffi.str2charp("version_info") - assert api.PySys_GetObject(version_info) - assert not api.PyErr_Occurred() - rffi.free_charp(version_info) +class AppTestSysModule(AppTestCpythonExtensionBase): + def test_sysmodule(self): + module = self.import_extension('foo', [ + ("get", "METH_VARARGS", + """ + char *name = PyString_AsString(PyTuple_GetItem(args, 0)); + PyObject *retval = PySys_GetObject(name); + return PyBool_FromLong(retval != NULL); + """)]) + assert module.get("excepthook") + assert not module.get("spam_spam_spam") - i_do_not_exist = rffi.str2charp("i_do_not_exist") - assert not api.PySys_GetObject(i_do_not_exist) - assert not api.PyErr_Occurred() - rffi.free_charp(i_do_not_exist) From afa at codespeak.net Wed Apr 7 18:25:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Apr 2010 18:25:18 +0200 (CEST) Subject: [pypy-svn] r73510 - in pypy/branch/cpython-extension/pypy/translator/c: . test Message-ID: <20100407162518.2230E282BD6@codespeak.net> Author: afa Date: Wed Apr 7 18:25:15 2010 New Revision: 73510 Modified: pypy/branch/cpython-extension/pypy/translator/c/genc.py pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py Log: Fix the test for --shared on Linux Modified: pypy/branch/cpython-extension/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/genc.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/genc.py Wed Apr 7 18:25:15 2010 @@ -79,8 +79,15 @@ self.profbased = profbased def _build(self, eci=ExternalCompilationInfo(), shared=False): + outputfilename = self.outputfilename + if shared: + if outputfilename: + basename = outputfilename + else: + basename = self.cfiles[0].purebasename + outputfilename = 'lib' + basename return self.platform.compile(self.cfiles, self.eci.merge(eci), - outputfilename=self.outputfilename, + outputfilename=outputfilename, standalone=not shared) def build(self, shared=False): @@ -468,26 +475,33 @@ return res.out, res.err return res.out - def build_main_for_shared(self, shared_library_name, entrypoint): + def build_main_for_shared(self, shared_library_name, entrypoint, exe_name): + import time + time.sleep(1) self.shared_library_name = shared_library_name # build main program eci = self.get_eci() + kw = {} + if self.translator.platform.so_ext == 'so': + kw['libraries'] = [self.shared_library_name.purebasename[3:]] + kw['library_dirs'] = [self.targetdir] + else: + kw['libraries'] = [self.shared_library_name.new(ext='')] eci = eci.merge(ExternalCompilationInfo( separate_module_sources=[''' - int %s(argc, argv); + int %s(int argc, char* argv[]); int main(int argc, char* argv[]) - { %s(argc, argv); } + { return %s(argc, argv); } ''' % (entrypoint, entrypoint) ], - libraries=[self.shared_library_name.new(ext='')] + **kw )) eci = eci.convert_sources_to_files( cache_dir=self.targetdir) - outfilename = self.shared_library_name.new(ext='') return self.translator.platform.compile( [], eci, - outputfilename=str(outfilename)) + outputfilename=exe_name) def compile(self, exe_name=None): assert self.c_source_filename @@ -510,7 +524,7 @@ self.executable_name = compiler.build(shared=shared) if shared: self.executable_name = self.build_main_for_shared( - self.executable_name, "pypy_main_startup") + self.executable_name, "pypy_main_startup", exe_name) assert self.executable_name self._compiled = True return self.executable_name Modified: pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/test/test_standalone.py Wed Apr 7 18:25:15 2010 @@ -589,7 +589,7 @@ # The traceback stops at f() because it's the first function that # captures the AssertionError, which makes the program abort. - def test_shared(self): + def test_shared(self, monkeypatch): def f(argv): print len(argv) def entry_point(argv): @@ -598,6 +598,8 @@ t, cbuilder = self.compile(entry_point, shared=True) assert cbuilder.shared_library_name is not None assert cbuilder.shared_library_name != cbuilder.executable_name + monkeypatch.setenv('LD_LIBRARY_PATH', + cbuilder.shared_library_name.dirpath()) out, err = cbuilder.cmdexec("a b") assert out == "3" From xoraxax at codespeak.net Wed Apr 7 18:51:34 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 18:51:34 +0200 (CEST) Subject: [pypy-svn] r73512 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407165134.13F0C282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 18:51:31 2010 New Revision: 73512 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Remove borrowed objects from the dict when they are deallocated and decref them in tests in case they are still alive after running the test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Wed Apr 7 18:51:31 2010 @@ -163,6 +163,10 @@ if obj.c_ob_refcnt == 0: state = space.fromcache(State) ptr = rffi.cast(ADDR, obj) + try: + del state.borrowed_objects[ptr] + except KeyError: + pass if ptr not in state.py_objects_r2w: w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) if space.is_w(w_type, space.w_str) or space.is_w(w_type, space.w_unicode): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Wed Apr 7 18:51:31 2010 @@ -196,6 +196,12 @@ for w_obj in state.non_heaptypes: Py_DecRef(self.space, w_obj) state.non_heaptypes[:] = [] + while state.borrowed_objects: + addr = state.borrowed_objects.keys()[0] + w_obj = state.py_objects_r2w[addr] + Py_DecRef(self.space, w_obj) + state.borrowed_objects = {} + state.borrow_mapping = {} except OperationError: pass except AttributeError: From xoraxax at codespeak.net Wed Apr 7 18:51:52 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 18:51:52 +0200 (CEST) Subject: [pypy-svn] r73513 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100407165152.A854D282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 18:51:50 2010 New Revision: 73513 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Insert gc_stack_bottom operation for trackgcroot.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 7 18:51:50 2010 @@ -26,6 +26,7 @@ # CPython 2.4 compatibility from py.builtin import BaseException from pypy.tool.sourcetools import func_with_new_name +from pypy.rpython.lltypesystem.lloperation import llop DEBUG_WRAPPER = False @@ -333,6 +334,9 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.pyobject import add_borrowed_object from pypy.module.cpyext.pyobject import NullPointerException + # we hope that malloc removal removes the newtuple() that is + # inserted exactly here by the varargs specializer + llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py boxed_args = () if DEBUG_WRAPPER: print >>sys.stderr, callable, From fijal at codespeak.net Wed Apr 7 19:17:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 19:17:43 +0200 (CEST) Subject: [pypy-svn] r73514 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100407171743.5DD68282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 19:17:38 2010 New Revision: 73514 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py Log: break translation - we should not see cpython bytecode during translation Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py Wed Apr 7 19:17:38 2010 @@ -117,6 +117,11 @@ self._compute_flatcall() + def _freeze_(self): + if self.magic == cpython_magic: + raise Exception("CPython host codes should not be rendered") + # XXX since PyCode is an immutable object, why not return True? + return False def _init_flags(self): co_code = self.co_code From xoraxax at codespeak.net Wed Apr 7 19:19:19 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 7 Apr 2010 19:19:19 +0200 (CEST) Subject: [pypy-svn] r73515 - pypy/branch/cpython-extension/pypy/translator/platform Message-ID: <20100407171919.E5878282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 7 19:19:18 2010 New Revision: 73515 Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py Log: Also call the gnu linker correctly with all external symbols when the linking is done without makefile. Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/posix.py Wed Apr 7 19:19:18 2010 @@ -36,6 +36,10 @@ cwd=str(cfile.dirpath())) return oname + def _link_args_from_eci(self, eci, standalone): + eci = eci.convert_exportsymbols_to_file() + return Platform._link_args_from_eci(self, eci, standalone) + def _link(self, cc, ofiles, link_args, standalone, exe_name): args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] From fijal at codespeak.net Wed Apr 7 19:25:03 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 19:25:03 +0200 (CEST) Subject: [pypy-svn] r73516 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100407172503.3C08D282B9C@codespeak.net> Author: fijal Date: Wed Apr 7 19:25:01 2010 New Revision: 73516 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py Log: A good reason - we can't mix frozen pbcs with instances Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pycode.py Wed Apr 7 19:25:01 2010 @@ -120,7 +120,6 @@ def _freeze_(self): if self.magic == cpython_magic: raise Exception("CPython host codes should not be rendered") - # XXX since PyCode is an immutable object, why not return True? return False def _init_flags(self): From fijal at codespeak.net Wed Apr 7 21:02:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:02:40 +0200 (CEST) Subject: [pypy-svn] r73517 - in pypy/branch/decouple-host-opcodes/pypy: interpreter interpreter/test translator Message-ID: <20100407190240.69F91282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:02:24 2010 New Revision: 73517 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py Log: Hack around so we don't actually create cpython objects from source, but instead we pass source around and compile it with our own compiler. Does not really solve the issue of calling "for internal debugging" space.exec_ for the translation Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Wed Apr 7 21:02:24 2010 @@ -13,7 +13,7 @@ from pypy.rlib.timer import DummyTimer, Timer from pypy.rlib.rarithmetic import r_uint from pypy.rlib import jit -import os, sys +import os, sys, py __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -948,12 +948,15 @@ raise TypeError, 'space.eval(): expected a string, code or PyCode object' return expression.exec_code(self, w_globals, w_locals) - def exec_(self, statement, w_globals, w_locals, hidden_applevel=False): + def exec_(self, statement, w_globals, w_locals, hidden_applevel=False, + filename=None): "NOT_RPYTHON: For internal debugging." import types + if filename is None: + filename = '?' from pypy.interpreter.pycode import PyCode if isinstance(statement, str): - statement = compile(statement, '?', 'exec') + statement = py.code.Source(statement).compile() if isinstance(statement, types.CodeType): statement = PyCode._from_code(self, statement, hidden_applevel=hidden_applevel) Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py Wed Apr 7 21:02:24 2010 @@ -821,12 +821,9 @@ hidden_applevel = True - def __init__(self, source, filename = None, modname = '__builtin__'): + def __init__(self, source, filename=None, modname='__builtin__'): self.filename = filename - if self.filename is None: - self.code = py.code.Source(source).compile() - else: - self.code = NiceCompile(self.filename)(source) + self.source = str(py.code.Source(source).deindent()) self.modname = modname # look at the first three lines for a NOT_RPYTHON tag first = "\n".join(source.split("\n", 3)[:3]) @@ -910,8 +907,9 @@ from pypy.interpreter.pycode import PyCode w_glob = space.newdict(module=True) space.setitem(w_glob, space.wrap('__name__'), space.wrap(self.modname)) - space.exec_(self.code, w_glob, w_glob, - hidden_applevel=self.hidden_applevel) + space.exec_(self.source, w_glob, w_glob, + hidden_applevel=self.hidden_applevel, + filename=self.filename) return w_glob # __________ geninterplevel version __________ @@ -931,7 +929,7 @@ from pypy.translator.geninterplevel import translate_as_module import marshal scramble = md5(cls.seed) - scramble.update(marshal.dumps(self.code)) + scramble.update(marshal.dumps(self.source)) key = scramble.hexdigest() initfunc = cls.known_code.get(key) if not initfunc: @@ -952,7 +950,7 @@ if not initfunc: # build it and put it into a file initfunc, newsrc = translate_as_module( - self.code, self.filename, self.modname) + self.source, self.filename, self.modname) fname = cls.cache_path.join(name+".py").strpath f = file(get_tmp_file_name(fname), "w") print >> f, """\ Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py Wed Apr 7 21:02:24 2010 @@ -1,5 +1,6 @@ from pypy.conftest import gettestobjspace from pypy.interpreter import gateway +from pypy.interpreter import baseobjspace import py class AppTestCodeIntrospection: @@ -9,7 +10,7 @@ if py.test.config.option.runappdirect: filename = __file__ else: - filename = gateway.__file__ + filename = baseobjspace.__file__ if filename[-3:] != '.py': filename = filename[:-1] Modified: pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py Wed Apr 7 21:02:24 2010 @@ -71,7 +71,7 @@ log = py.log.Producer("geninterp") py.log.setconsumer("geninterp", ansi_log) -GI_VERSION = '1.2.7' # bump this for substantial changes +GI_VERSION = '1.2.8' # bump this for substantial changes # ____________________________________________________________ try: @@ -1475,10 +1475,7 @@ """ # create something like a module if type(sourcetext) is str: - if filename is None: - code = py.code.Source(sourcetext).compile() - else: - code = NiceCompile(filename)(sourcetext) + code = py.code.Source(sourcetext).compile() else: # assume we got an already compiled source code = sourcetext From fijal at codespeak.net Wed Apr 7 21:06:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:06:00 +0200 (CEST) Subject: [pypy-svn] r73518 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100407190600.7D10D282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:05:44 2010 New Revision: 73518 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/nanos.py Log: fix translation Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/nanos.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/nanos.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/nanos.py Wed Apr 7 21:05:44 2010 @@ -26,7 +26,7 @@ """ from pypy.interpreter.gateway import applevel, ObjSpace, W_Root, interp2app -import os +import os, py app_os_path = applevel(r''' from os.path import dirname, join, abspath, isfile, islink @@ -59,6 +59,6 @@ path_module_for_testing = type(os)("os.path") os_module_for_testing = type(os)("os") os_module_for_testing.path = path_module_for_testing -eval(app_os_path.code, path_module_for_testing.__dict__) -eval(app_os.code, os_module_for_testing.__dict__) +eval(py.code.Source(app_os_path.source).compile(), path_module_for_testing.__dict__) +eval(py.code.Source(app_os.source).compile(), os_module_for_testing.__dict__) From fijal at codespeak.net Wed Apr 7 21:29:26 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:29:26 +0200 (CEST) Subject: [pypy-svn] r73519 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100407192926.EBEE9282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:29:22 2010 New Revision: 73519 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Log: Don't use space.exec_ here Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Wed Apr 7 21:29:22 2010 @@ -233,8 +233,8 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict() - space.exec_(open(filename).read(), w_dict, w_dict) + app = gatway.applevel(open(filename).read(), 'app_main.py', 'app_main') + w_dict = app.getwdict() entry_point = create_entry_point(space, w_dict) return entry_point, None, PyPyAnnotatorPolicy(single_space = space) From fijal at codespeak.net Wed Apr 7 21:32:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:32:11 +0200 (CEST) Subject: [pypy-svn] r73520 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100407193211.59449282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:32:09 2010 New Revision: 73520 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Log: wonderful joys of untested code Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Wed Apr 7 21:32:09 2010 @@ -233,8 +233,8 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - app = gatway.applevel(open(filename).read(), 'app_main.py', 'app_main') - w_dict = app.getwdict() + app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + w_dict = app.getwdict(space) entry_point = create_entry_point(space, w_dict) return entry_point, None, PyPyAnnotatorPolicy(single_space = space) From fijal at codespeak.net Wed Apr 7 21:38:39 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:38:39 +0200 (CEST) Subject: [pypy-svn] r73521 - pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2 Message-ID: <20100407193839.EF94C282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:38:38 2010 New Revision: 73521 Added: pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py (contents, props changed) Log: start writing tests for targetpypy Added: pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py ============================================================================== --- (empty file) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py Wed Apr 7 21:38:38 2010 @@ -0,0 +1,8 @@ + +from pypy.translator.goal.targetpypystandalone import get_entry_point +from pypy.config.pypyoption import get_pypy_config + +class TestTargetPyPy(object): + def test_run(self): + config = get_pypy_config(translating=True) + entry_point = get_entry_point(config) From fijal at codespeak.net Wed Apr 7 21:40:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:40:16 +0200 (CEST) Subject: [pypy-svn] r73522 - pypy/branch/decouple-host-opcodes/pypy/objspace/flow Message-ID: <20100407194016.01BF0282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:40:15 2010 New Revision: 73522 Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/specialcase.py Log: fix objspace/flow tests Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/specialcase.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/specialcase.py Wed Apr 7 21:40:15 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import ApplevelClass from pypy.interpreter.error import OperationError from pypy.tool.cache import Cache +import py def sc_import(space, fn, args): args_w, kwds_w = args.unpack() @@ -73,7 +74,7 @@ if app.filename is not None: dic['__file__'] = app.filename dic['__name__'] = app.modname - exec app.code in dic + exec py.code.Source(app.source).compile() in dic return dic _build = staticmethod(_build) From fijal at codespeak.net Wed Apr 7 21:41:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:41:17 +0200 (CEST) Subject: [pypy-svn] r73523 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100407194117.4CBED282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:41:15 2010 New Revision: 73523 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Log: that belongs to previous commit Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Wed Apr 7 21:41:15 2010 @@ -241,7 +241,7 @@ def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', - 'jitpolicy', + 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) From fijal at codespeak.net Wed Apr 7 21:51:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:51:23 +0200 (CEST) Subject: [pypy-svn] r73524 - pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2 Message-ID: <20100407195123.AB254282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:51:22 2010 New Revision: 73524 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py Log: A test that I would like to work Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py Wed Apr 7 21:51:22 2010 @@ -1,8 +1,12 @@ +import py from pypy.translator.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): def test_run(self): - config = get_pypy_config(translating=True) - entry_point = get_entry_point(config) + config = get_pypy_config(translating=False) + entry_point = get_entry_point(config)[0] + space = self.space + py.test.skip("not working so far") + entry_point(['pypy-c' , '-S', '-c', 'print 3']) From fijal at codespeak.net Wed Apr 7 21:52:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 21:52:11 +0200 (CEST) Subject: [pypy-svn] r73525 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100407195211.52888282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 21:52:09 2010 New Revision: 73525 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Log: Don't use geninterp for app_main Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Wed Apr 7 21:52:09 2010 @@ -234,6 +234,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + app.can_use_geninterp = False w_dict = app.getwdict(space) entry_point = create_entry_point(space, w_dict) From fijal at codespeak.net Wed Apr 7 23:19:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 23:19:46 +0200 (CEST) Subject: [pypy-svn] r73526 - pypy/branch/decouple-host-opcodes/pypy/module/_codecs Message-ID: <20100407211946.C4988282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 23:19:45 2010 New Revision: 73526 Modified: pypy/branch/decouple-host-opcodes/pypy/module/_codecs/app_codecs.py Log: Kill contradictory comment. Make this RPython and not blow up with uninitialized variable Modified: pypy/branch/decouple-host-opcodes/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/module/_codecs/app_codecs.py Wed Apr 7 23:19:45 2010 @@ -1,7 +1,3 @@ -# NOT_RPYTHON -# Note: -# This *is* now explicitly RPython. -# Please make sure not to break this. """ @@ -356,6 +352,7 @@ bitsleft = 0 charsleft = 0 surrogate = 0 + startinpos = 0 p = [] errorHandler = None exc = None From fijal at codespeak.net Wed Apr 7 23:50:51 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Apr 2010 23:50:51 +0200 (CEST) Subject: [pypy-svn] r73527 - in pypy/branch/decouple-host-opcodes/pypy: interpreter interpreter/test tool/pytest Message-ID: <20100407215051.7EFFC282BD6@codespeak.net> Author: fijal Date: Wed Apr 7 23:50:49 2010 New Revision: 73527 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_appinterp.py pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py Log: Try slightly harder to compile our own code with our own compiler. Fail at geninterp so far Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Wed Apr 7 23:50:49 2010 @@ -956,7 +956,8 @@ filename = '?' from pypy.interpreter.pycode import PyCode if isinstance(statement, str): - statement = py.code.Source(statement).compile() + compiler = self.createcompiler() + statement = compiler.compile(statement, filename, 'exec', 0) if isinstance(statement, types.CodeType): statement = PyCode._from_code(self, statement, hidden_applevel=hidden_applevel) @@ -1177,7 +1178,7 @@ assert source.startswith('('), "incorrect header in:\n%s" % (source,) source = py.code.Source("def anonymous%s\n" % source) w_glob = space.newdict() - space.exec_(source.compile(), w_glob, w_glob) + space.exec_(str(source), w_glob, w_glob) return space.getitem(w_glob, space.wrap('anonymous')) class DummyLock(object): Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py Wed Apr 7 23:50:49 2010 @@ -812,6 +812,9 @@ # and now for something completely different ... # +class MyStr(str): + pass + class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -822,6 +825,11 @@ hidden_applevel = True def __init__(self, source, filename=None, modname='__builtin__'): + # HAAACK (but a good one) + if filename is None: + f = sys._getframe(2) + filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) + filename.__source__ = py.code.Source(source) self.filename = filename self.source = str(py.code.Source(source).deindent()) self.modname = modname Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_appinterp.py Wed Apr 7 23:50:49 2010 @@ -1,6 +1,7 @@ import py from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp, applevelinterp_temp +from pypy.interpreter.error import OperationError def test_execwith_novars(space): val = space.appexec([], """ @@ -18,11 +19,11 @@ assert space.eq_w(val, space.wrap(42)) def test_execwith_compile_error(space): - excinfo = py.test.raises(SyntaxError, space.appexec, [], """ + excinfo = py.test.raises(OperationError, space.appexec, [], """ (): y y """) - assert str(excinfo.value).find('y y') != -1 + assert str(excinfo.value.errorstr(space)).find('y y') != -1 def test_simple_applevel(space): app = appdef("""app(x,y): Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py Wed Apr 7 23:50:49 2010 @@ -10,7 +10,7 @@ if py.test.config.option.runappdirect: filename = __file__ else: - filename = baseobjspace.__file__ + filename = gateway.__file__ if filename[-3:] != '.py': filename = filename[:-1] Modified: pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py Wed Apr 7 23:50:49 2010 @@ -87,7 +87,7 @@ if debug_excs: self._excinfo = debug_excs[0] - def exconly(self, tryshort=True): + def exconly(self, tryshort=True): return '(application-level) ' + self.operr.errorstr(self.space) def errisinstance(self, exc): From fijal at codespeak.net Thu Apr 8 00:01:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Apr 2010 00:01:14 +0200 (CEST) Subject: [pypy-svn] r73528 - pypy/branch/decouple-host-opcodes/pypy/translator Message-ID: <20100407220114.4B130282BD6@codespeak.net> Author: fijal Date: Thu Apr 8 00:01:12 2010 New Revision: 73528 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/driver.py Log: Sprinkle a bit of doctests Modified: pypy/branch/decouple-host-opcodes/pypy/translator/driver.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/driver.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/driver.py Thu Apr 8 00:01:12 2010 @@ -294,6 +294,8 @@ return res def task_annotate(self): + """ Annotate + """ # includes annotation and annotatation simplifications translator = self.translator policy = self.policy @@ -349,6 +351,8 @@ def task_rtype_lltype(self): + """ RTyping - lltype version + """ rtyper = self.translator.buildrtyper(type_system='lltype') insist = not self.config.translation.insist rtyper.specialize(dont_simplify_again=True, @@ -358,6 +362,8 @@ RTYPE = 'rtype_lltype' def task_rtype_ootype(self): + """ RTyping - ootype version + """ # Maybe type_system should simply be an option used in task_rtype insist = not self.config.translation.insist rtyper = self.translator.buildrtyper(type_system="ootype") @@ -368,6 +374,9 @@ OOTYPE = 'rtype_ootype' def task_pyjitpl_lltype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -383,6 +392,9 @@ "JIT compiler generation") def task_pyjitpl_ootype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -398,6 +410,8 @@ "JIT compiler generation") def task_backendopt_lltype(self): + """ Run all backend optimizations - lltype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -407,6 +421,8 @@ BACKENDOPT = 'backendopt_lltype' def task_backendopt_ootype(self): + """ Run all backend optimizations - ootype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -437,6 +453,8 @@ raise Exception(str(e) + '\n' + i) def task_database_c(self): + """ Create a database for further backend generation + """ translator = self.translator if translator.annotator is not None: translator.frozen = True @@ -467,7 +485,9 @@ "Creating database for generating c source", earlycheck = possibly_check_for_boehm) - def task_source_c(self): # xxx messy + def task_source_c(self): + """ Create C source files from the generated database + """ translator = self.translator cbuilder = self.cbuilder database = self.database @@ -495,6 +515,8 @@ return mkexename(py.path.local(newexename)) def create_exe(self): + """ Copy the compiled executable into translator/goal + """ if self.exe_name is not None: exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() @@ -502,7 +524,10 @@ self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) - def task_compile_c(self): # xxx messy + def task_compile_c(self): + """ Compile the generated C code using either makefile or + translator/platform + """ cbuilder = self.cbuilder cbuilder.compile() From fijal at codespeak.net Thu Apr 8 00:03:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Apr 2010 00:03:18 +0200 (CEST) Subject: [pypy-svn] r73529 - pypy/trunk/pypy/translator Message-ID: <20100407220318.D594C282BD6@codespeak.net> Author: fijal Date: Thu Apr 8 00:03:17 2010 New Revision: 73529 Modified: pypy/trunk/pypy/translator/driver.py Log: Sprinkle a bit of doctests Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Thu Apr 8 00:03:17 2010 @@ -294,6 +294,8 @@ return res def task_annotate(self): + """ Annotate + """ # includes annotation and annotatation simplifications translator = self.translator policy = self.policy @@ -349,6 +351,8 @@ def task_rtype_lltype(self): + """ RTyping - lltype version + """ rtyper = self.translator.buildrtyper(type_system='lltype') insist = not self.config.translation.insist rtyper.specialize(dont_simplify_again=True, @@ -358,6 +362,8 @@ RTYPE = 'rtype_lltype' def task_rtype_ootype(self): + """ RTyping - ootype version + """ # Maybe type_system should simply be an option used in task_rtype insist = not self.config.translation.insist rtyper = self.translator.buildrtyper(type_system="ootype") @@ -368,6 +374,9 @@ OOTYPE = 'rtype_ootype' def task_pyjitpl_lltype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -383,6 +392,9 @@ "JIT compiler generation") def task_pyjitpl_ootype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -398,6 +410,8 @@ "JIT compiler generation") def task_backendopt_lltype(self): + """ Run all backend optimizations - lltype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -407,6 +421,8 @@ BACKENDOPT = 'backendopt_lltype' def task_backendopt_ootype(self): + """ Run all backend optimizations - ootype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -437,6 +453,8 @@ raise Exception(str(e) + '\n' + i) def task_database_c(self): + """ Create a database for further backend generation + """ translator = self.translator if translator.annotator is not None: translator.frozen = True @@ -467,7 +485,9 @@ "Creating database for generating c source", earlycheck = possibly_check_for_boehm) - def task_source_c(self): # xxx messy + def task_source_c(self): + """ Create C source files from the generated database + """ translator = self.translator cbuilder = self.cbuilder database = self.database @@ -495,6 +515,8 @@ return mkexename(py.path.local(newexename)) def create_exe(self): + """ Copy the compiled executable into translator/goal + """ if self.exe_name is not None: exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() @@ -502,7 +524,10 @@ self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) - def task_compile_c(self): # xxx messy + def task_compile_c(self): + """ Compile the generated C code using either makefile or + translator/platform + """ cbuilder = self.cbuilder cbuilder.compile() From trundle at codespeak.net Thu Apr 8 00:35:22 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 8 Apr 2010 00:35:22 +0200 (CEST) Subject: [pypy-svn] r73531 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100407223522.86880282BD6@codespeak.net> Author: trundle Date: Thu Apr 8 00:35:20 2010 New Revision: 73531 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add Py_UNICODE_TOUPPER. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Thu Apr 8 00:35:20 2010 @@ -5666,11 +5666,6 @@ raise NotImplementedError @cpython_api([{Py_UNICODE}], {Py_UNICODE}) -def Py_UNICODE_TOUPPER(space, ch): - """Return the character ch converted to upper case.""" - raise NotImplementedError - - at cpython_api([{Py_UNICODE}], {Py_UNICODE}) def Py_UNICODE_TOTITLE(space, ch): """Return the character ch converted to title case.""" raise NotImplementedError Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Thu Apr 8 00:35:20 2010 @@ -48,3 +48,6 @@ assert api.Py_UNICODE_TOLOWER(u'?') == u'?' assert api.Py_UNICODE_TOLOWER(u'?') == u'?' + def test_TOUPPER(self, space, api): + assert api.Py_UNICODE_TOUPPER(u'?') == u'?' + assert api.Py_UNICODE_TOUPPER(u'?') == u'?' Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Thu Apr 8 00:35:20 2010 @@ -55,6 +55,11 @@ """Return the character ch converted to lower case.""" return unichr(unicodedb.tolower(ord(ch))) + at cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL) +def Py_UNICODE_TOUPPER(space, ch): + """Return the character ch converted to upper case.""" + return unichr(unicodedb.toupper(ord(ch))) + @cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyUnicode_AS_DATA(space, ref): """Return a pointer to the internal buffer of the object. o has to be a From benjamin at codespeak.net Thu Apr 8 03:35:28 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 8 Apr 2010 03:35:28 +0200 (CEST) Subject: [pypy-svn] r73532 - in pypy/trunk/pypy: module/signal translator/c/src Message-ID: <20100408013528.BD8FB282BD6@codespeak.net> Author: benjamin Date: Thu Apr 8 03:35:21 2010 New Revision: 73532 Modified: pypy/trunk/pypy/module/signal/interp_signal.py pypy/trunk/pypy/translator/c/src/signals.h Log: use a struct for pypysig_occurred in signals.h This kills some strict aliasing warnings. Modified: pypy/trunk/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/trunk/pypy/module/signal/interp_signal.py (original) +++ pypy/trunk/pypy/module/signal/interp_signal.py Thu Apr 8 03:35:21 2010 @@ -39,7 +39,10 @@ # pointless and a performance issue # don't use rffi.LONGP because the JIT doesn't support raw arrays so far -LONG_STRUCT = lltype.Struct('LONG_STRUCT', ('value', lltype.Signed)) +struct_name = 'pypysig_long_struct' +LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), + hints={'c_name' : struct_name, 'external' : 'C'}) +del struct_name pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], lltype.Ptr(LONG_STRUCT), _nowrapper=True, @@ -51,10 +54,10 @@ class SignalActionFlag(AbstractActionFlag): def get(self): p = pypysig_getaddr_occurred() - return p.value + return p.c_value def set(self, value): p = pypysig_getaddr_occurred() - p.value = value + p.c_value = value class CheckSignalAction(AsyncAction): Modified: pypy/trunk/pypy/translator/c/src/signals.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/signals.h (original) +++ pypy/trunk/pypy/translator/c/src/signals.h Thu Apr 8 03:35:21 2010 @@ -70,7 +70,11 @@ cleared again. The variable is exposed and RPython code is free to use the other bits in any way. */ #define PENDING_SIGNAL_BIT (LONG_MIN) /* high bit */ -extern long pypysig_occurred; +/* This is a struct for the JIT. See interp_signal.py. */ +struct pypysig_long_struct { + long value; +}; +extern struct pypysig_long_struct pypysig_occurred; /* some C tricks to get/set the variable as efficiently as possible: use macros when compiling as a stand-alone program, but still @@ -87,9 +91,8 @@ #ifndef PYPY_NOT_MAIN_FILE - -long pypysig_occurred; -static volatile long *pypysig_occurred_v = (volatile long *)&pypysig_occurred; +struct pypysig_long_struct pypysig_occurred; +static volatile long *pypysig_occurred_v = (volatile long *)&pypysig_occurred.value; static volatile int pypysig_flags[NSIG]; void pypysig_ignore(int signum) @@ -159,7 +162,7 @@ { pypysig_flags[i] = 0; /* maybe another signal is pending: */ - pypysig_occurred |= PENDING_SIGNAL_BIT; + pypysig_occurred.value |= PENDING_SIGNAL_BIT; return i; } } From afa at codespeak.net Thu Apr 8 09:14:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 09:14:23 +0200 (CEST) Subject: [pypy-svn] r73533 - pypy/trunk/pypy/translator/platform Message-ID: <20100408071423.19431282BD6@codespeak.net> Author: afa Date: Thu Apr 8 09:14:21 2010 New Revision: 73533 Modified: pypy/trunk/pypy/translator/platform/maemo.py pypy/trunk/pypy/translator/platform/windows.py Log: fix execute_makefile for non-posix platforms. Modified: pypy/trunk/pypy/translator/platform/maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/maemo.py (original) +++ pypy/trunk/pypy/translator/platform/maemo.py Thu Apr 8 09:14:21 2010 @@ -82,11 +82,11 @@ # on the other hand, library lands in usual place... return [] - def execute_makefile(self, path_to_makefile): + def execute_makefile(self, path_to_makefile, extra_opts=[]): if isinstance(path_to_makefile, GnuMakefile): path = path_to_makefile.makefile_dir else: path = path_to_makefile - log.execute('make in %s' % (path,)) - returncode, stdout, stderr = _run_subprocess('/scratchbox/login', ['make', '-C', str(path)]) + log.execute('make %s in %s' % (" ".join(extra_opts), path)) + returncode, stdout, stderr = _run_subprocess('/scratchbox/login', ['make', '-C', str(path)] + extra_opts) self._handle_error(returncode, stdout, stderr, path.join('make')) Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Thu Apr 8 09:14:21 2010 @@ -255,17 +255,17 @@ return m - def execute_makefile(self, path_to_makefile): + def execute_makefile(self, path_to_makefile, extra_opts=[]): if isinstance(path_to_makefile, NMakefile): path = path_to_makefile.makefile_dir else: path = path_to_makefile - log.execute('make in %s' % (path,)) + log.execute('make %s in %s' % (" ".join(extra_opts), path)) oldcwd = path.chdir() try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/nologo', '/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts) finally: oldcwd.chdir() From afa at codespeak.net Thu Apr 8 09:15:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 09:15:30 +0200 (CEST) Subject: [pypy-svn] r73534 - pypy/trunk/pypy/translator/platform Message-ID: <20100408071530.128EF282BD6@codespeak.net> Author: afa Date: Thu Apr 8 09:15:28 2010 New Revision: 73534 Modified: pypy/trunk/pypy/translator/platform/maemo.py pypy/trunk/pypy/translator/platform/posix.py Log: wrap long lines Modified: pypy/trunk/pypy/translator/platform/maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/maemo.py (original) +++ pypy/trunk/pypy/translator/platform/maemo.py Thu Apr 8 09:15:28 2010 @@ -88,5 +88,6 @@ else: path = path_to_makefile log.execute('make %s in %s' % (" ".join(extra_opts), path)) - returncode, stdout, stderr = _run_subprocess('/scratchbox/login', ['make', '-C', str(path)] + extra_opts) + returncode, stdout, stderr = _run_subprocess( + '/scratchbox/login', ['make', '-C', str(path)] + extra_opts) self._handle_error(returncode, stdout, stderr, path.join('make')) Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Thu Apr 8 09:15:28 2010 @@ -123,7 +123,8 @@ else: path = path_to_makefile log.execute('make %s in %s' % (" ".join(extra_opts), path)) - returncode, stdout, stderr = _run_subprocess(self.make_cmd, ['-C', str(path)] + extra_opts) + returncode, stdout, stderr = _run_subprocess( + self.make_cmd, ['-C', str(path)] + extra_opts) self._handle_error(returncode, stdout, stderr, path.join('make')) class Definition(object): From afa at codespeak.net Thu Apr 8 09:25:40 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 09:25:40 +0200 (CEST) Subject: [pypy-svn] r73535 - pypy/branch/cpython-extension/pypy/translator/platform Message-ID: <20100408072540.9FF3D282BD6@codespeak.net> Author: afa Date: Thu Apr 8 09:25:39 2010 New Revision: 73535 Modified: pypy/branch/cpython-extension/pypy/translator/platform/maemo.py pypy/branch/cpython-extension/pypy/translator/platform/posix.py pypy/branch/cpython-extension/pypy/translator/platform/windows.py Log: merge -r73532-73534 from trunk Modified: pypy/branch/cpython-extension/pypy/translator/platform/maemo.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/maemo.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/maemo.py Thu Apr 8 09:25:39 2010 @@ -82,11 +82,12 @@ # on the other hand, library lands in usual place... return [] - def execute_makefile(self, path_to_makefile): + def execute_makefile(self, path_to_makefile, extra_opts=[]): if isinstance(path_to_makefile, GnuMakefile): path = path_to_makefile.makefile_dir else: path = path_to_makefile - log.execute('make in %s' % (path,)) - returncode, stdout, stderr = _run_subprocess('/scratchbox/login', ['make', '-C', str(path)]) + log.execute('make %s in %s' % (" ".join(extra_opts), path)) + returncode, stdout, stderr = _run_subprocess( + '/scratchbox/login', ['make', '-C', str(path)] + extra_opts) self._handle_error(returncode, stdout, stderr, path.join('make')) Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/posix.py Thu Apr 8 09:25:39 2010 @@ -128,7 +128,8 @@ else: path = path_to_makefile log.execute('make %s in %s' % (" ".join(extra_opts), path)) - returncode, stdout, stderr = _run_subprocess(self.make_cmd, ['-C', str(path)] + extra_opts) + returncode, stdout, stderr = _run_subprocess( + self.make_cmd, ['-C', str(path)] + extra_opts) self._handle_error(returncode, stdout, stderr, path.join('make')) class Definition(object): Modified: pypy/branch/cpython-extension/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/windows.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/windows.py Thu Apr 8 09:25:39 2010 @@ -255,17 +255,17 @@ return m - def execute_makefile(self, path_to_makefile): + def execute_makefile(self, path_to_makefile, extra_opts=[]): if isinstance(path_to_makefile, NMakefile): path = path_to_makefile.makefile_dir else: path = path_to_makefile - log.execute('make in %s' % (path,)) + log.execute('make %s in %s' % (" ".join(extra_opts), path)) oldcwd = path.chdir() try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/nologo', '/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts) finally: oldcwd.chdir() From jandem at codespeak.net Thu Apr 8 11:00:29 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 11:00:29 +0200 (CEST) Subject: [pypy-svn] r73536 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408090029.4EA03282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 11:00:27 2010 New Revision: 73536 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: add PyErr_BadArgument Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Thu Apr 8 11:00:27 2010 @@ -20,6 +20,13 @@ message = rffi.charp2str(message_ptr) PyErr_SetObject(space, w_type, space.wrap(message)) + at cpython_api([], lltype.Void) +def PyErr_BadArgument(space): + """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where + message indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use.""" + raise OperationError(space.w_TypeError, + space.wrap("bad argument type for built-in operation")) @cpython_api([PyObject], PyObject) def PyErr_SetFromErrno(space, w_type): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Thu Apr 8 11:00:27 2010 @@ -64,7 +64,12 @@ api.PyErr_NoMemory() assert space.eq_w(state.exc_type, space.w_MemoryError) api.PyErr_Clear() - + + def test_BadArgument(self, space, api): + api.PyErr_BadArgument() + state = space.fromcache(State) + assert space.eq_w(state.exc_type, space.w_TypeError) + api.PyErr_Clear() class AppTestFetch(AppTestCpythonExtensionBase): def setup_class(cls): From jandem at codespeak.net Thu Apr 8 11:29:16 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 11:29:16 +0200 (CEST) Subject: [pypy-svn] r73537 - in pypy/branch/cpython-extension/pypy/module/cpyext: include src Message-ID: <20100408092916.9F1C3282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 11:29:15 2010 New Revision: 73537 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c Log: add PyString_FromFormat Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Thu Apr 8 11:29:15 2010 @@ -17,6 +17,7 @@ } PyStringObject; PyObject *PyString_FromFormatV(char *format, va_list vargs); +PyObject *PyString_FromFormat(const char *format, ...); #ifdef __cplusplus } Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/stringobject.c Thu Apr 8 11:29:15 2010 @@ -231,3 +231,19 @@ _PyString_Resize(&string, s - PyString_AS_STRING(string)); return string; } + +PyObject * +PyString_FromFormat(const char *format, ...) +{ + PyObject* ret; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + ret = PyString_FromFormatV(format, vargs); + va_end(vargs); + return ret; +} From jandem at codespeak.net Thu Apr 8 11:38:24 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 11:38:24 +0200 (CEST) Subject: [pypy-svn] r73538 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408093824.DB9FE282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 11:38:23 2010 New Revision: 73538 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Add test for PyString_FromFormat and fix FormatV test Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 11:38:23 2010 @@ -226,8 +226,8 @@ FUNCTIONS_C = [ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', - 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', - 'PyTuple_Pack', 'PyErr_Format', + 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', + 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Thu Apr 8 11:38:23 2010 @@ -125,7 +125,20 @@ } ''') res = module.test_string_format_v(1, "xyz") - print res + assert res == "bla 1 ble xyz\n" + + def test_format(self): + module = self.import_extension('foo', [ + ("test_string_format", "METH_VARARGS", + ''' + return PyString_FromFormat("bla %d ble %s\\n", + PyInt_AsLong(PyTuple_GetItem(args, 0)), + PyString_AsString(PyTuple_GetItem(args, 1))); + ''' + ) + ]) + res = module.test_string_format(1, "xyz") + assert res == "bla 1 ble xyz\n" class TestString(BaseApiTest): def test_string_resize(self, space, api): From pedronis at codespeak.net Thu Apr 8 13:41:18 2010 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 8 Apr 2010 13:41:18 +0200 (CEST) Subject: [pypy-svn] r73541 - pypy/build/testrunner Message-ID: <20100408114118.13CE4282BD6@codespeak.net> Author: pedronis Date: Thu Apr 8 13:41:16 2010 New Revision: 73541 Modified: pypy/build/testrunner/runner.py Log: revert last checkin to redo it with a proper log entry Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Thu Apr 8 13:41:16 2010 @@ -362,6 +362,7 @@ if py.path.local(config_py_file).check(file=1): print >>out, "using config", config_py_file execfile(config_py_file, run_param.__dict__) + run_param.__dict__.pop('__builtins__', None) if run_param.cherrypick: for p in run_param.cherrypick: From pedronis at codespeak.net Thu Apr 8 13:41:36 2010 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 8 Apr 2010 13:41:36 +0200 (CEST) Subject: [pypy-svn] r73542 - pypy/build/testrunner Message-ID: <20100408114136.60736282BD6@codespeak.net> Author: pedronis Date: Thu Apr 8 13:41:34 2010 New Revision: 73542 Modified: pypy/build/testrunner/runner.py Log: allow configs to use builtins Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Thu Apr 8 13:41:34 2010 @@ -362,7 +362,6 @@ if py.path.local(config_py_file).check(file=1): print >>out, "using config", config_py_file execfile(config_py_file, run_param.__dict__) - run_param.__dict__.pop('__builtins__', None) if run_param.cherrypick: for p in run_param.cherrypick: From afa at codespeak.net Thu Apr 8 14:52:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 14:52:54 +0200 (CEST) Subject: [pypy-svn] r73544 - in pypy/branch/cpython-extension/pypy/translator: c c/gcc c/gcc/test platform Message-ID: <20100408125254.113A1282BD6@codespeak.net> Author: afa Date: Thu Apr 8 14:52:53 2010 New Revision: 73544 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py pypy/branch/cpython-extension/pypy/translator/c/genc.py pypy/branch/cpython-extension/pypy/translator/platform/__init__.py pypy/branch/cpython-extension/pypy/translator/platform/posix.py pypy/branch/cpython-extension/pypy/translator/platform/windows.py Log: Adapt makefile and trackgcroot to the --shared option. On Windows for the moment Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Apr 8 14:52:53 2010 @@ -186,6 +186,13 @@ def define_callback_with_collect(cls): return lambda: 0 +class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC): + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.shared = True + return config + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py Thu Apr 8 14:52:53 2010 @@ -1613,6 +1613,7 @@ format = 'mingw32' else: format = 'elf' + entrypoint = 'main' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1626,6 +1627,9 @@ elif sys.argv[1].startswith('-f'): format = sys.argv[1][2:] del sys.argv[1] + elif sys.argv[1].startswith('-m'): + entrypoint = sys.argv[1][2:] + del sys.argv[1] else: break tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) @@ -1642,7 +1646,7 @@ lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: - tracker.process(f, g, filename=fn) + tracker.process(f, g, entrypoint=entrypoint, filename=fn) except: g.close() os.unlink(lblfn) Modified: pypy/branch/cpython-extension/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/genc.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/genc.py Thu Apr 8 14:52:53 2010 @@ -428,7 +428,7 @@ if isinstance(self._module, isolate.Isolate): isolate.close_isolate(self._module) - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): pass class CStandaloneBuilder(CBuilder): @@ -531,9 +531,11 @@ def gen_makefile(self, targetdir, exe_name=None): cfiles = [self.c_source_filename] + self.extrafiles - mk = self.translator.platform.gen_makefile(cfiles, self.eci, - path=targetdir, - exe_name=exe_name) + mk = self.translator.platform.gen_makefile( + cfiles, self.eci, + path=targetdir, exe_name=exe_name, + shared=self.config.translation.shared) + if self.has_profopt(): profopt = self.config.translation.profopt mk.definition('ABS_TARGET', '$(shell python -c "import sys,os; print os.path.abspath(sys.argv[1])" $(TARGET))') @@ -577,6 +579,11 @@ mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') + if self.config.translation.shared: + mk.definition('PYPY_MAIN_FUNCTION', "pypy_main_startup") + else: + mk.definition('PYPY_MAIN_FUNCTION', "main") + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: @@ -602,7 +609,7 @@ 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -m$(PYPY_MAIN_FUNCTION) -t $*.s > $@'] ) mk.rule('gcmaptable.c', '$(GCMAPFILES)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') @@ -611,7 +618,7 @@ mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -m$(PYPY_MAIN_FUNCTION) -t $< > $*.gcmap') mk.rule('gcmaptable.s', '$(GCMAPFILES)', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') Modified: pypy/branch/cpython-extension/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/__init__.py Thu Apr 8 14:52:53 2010 @@ -70,7 +70,8 @@ env) return ExecutionResult(returncode, stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): raise NotImplementedError("Pure abstract baseclass") def __repr__(self): Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/posix.py Thu Apr 8 14:52:53 2010 @@ -59,7 +59,8 @@ # strip compiler flags return [entry[2:] for entry in out.split()] - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): eci = eci.convert_exportsymbols_to_file() cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -72,6 +73,10 @@ if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci @@ -102,7 +107,7 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), Modified: pypy/branch/cpython-extension/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/windows.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/windows.py Thu Apr 8 14:52:53 2010 @@ -186,7 +186,8 @@ raise CompilationError(stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -202,6 +203,17 @@ m.exe_name = exe_name m.eci = eci + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + [ + '/EXPORT:$(PYPY_MAIN_FUNCTION)'] + + if shared: + so_name = exe_name.new(ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + def pypyrel(fpath): rel = py.path.local(fpath).relto(pypypath) if rel: @@ -218,8 +230,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -227,12 +239,13 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', self.link), ('MASM', self.masm), ] + for args in definitions: m.definition(*args) @@ -253,6 +266,16 @@ 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) + if shared: + m.definition('SHARED_IMPORT_LIB', so_name.new(ext='lib').basename), + m.rule('main.c', '', + 'echo ' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], + '$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): From jandem at codespeak.net Thu Apr 8 14:59:36 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 14:59:36 +0200 (CEST) Subject: [pypy-svn] r73545 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408125936.B3A4B282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 14:59:35 2010 New Revision: 73545 Added: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: add PyIndex_Check Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 14:59:35 2010 @@ -54,6 +54,7 @@ import pypy.module.cpyext.unicodeobject import pypy.module.cpyext.pycobject import pypy.module.cpyext.sysmodule +import pypy.module.cpyext.number # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Thu Apr 8 14:59:35 2010 @@ -0,0 +1,15 @@ +from pypy.interpreter.error import OperationError +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.pyobject import PyObject +from pypy.rpython.lltypesystem import rffi, lltype + + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyIndex_Check(space, w_obj): + """Returns True if o is an index integer (has the nb_index slot of the + tp_as_number structure filled in). + """ + try: + space.index(w_obj) + return 1 + except OperationError: + return 0 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Thu Apr 8 14:59:35 2010 @@ -0,0 +1,9 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext import sequence + +class TestIterator(BaseApiTest): + def test_index(self, space, api): + assert api.PyIndex_Check(space.wrap(12)) + assert not api.PyIndex_Check(space.wrap('12')) From xoraxax at codespeak.net Thu Apr 8 15:17:41 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 15:17:41 +0200 (CEST) Subject: [pypy-svn] r73546 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100408131741.A29A2282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 15:17:40 2010 New Revision: 73546 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Amended Python.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Thu Apr 8 15:17:40 2010 @@ -45,9 +45,17 @@ #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) +// obviously wrong #define SIZEOF_SIZE_T 4 #define SIZEOF_LONG 4 +// from pyport.h +#ifdef SIZE_MAX +#define PY_SIZE_MAX SIZE_MAX +#else +#define PY_SIZE_MAX ((size_t)-1) +#endif + /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ #ifdef __CHAR_UNSIGNED__ From xoraxax at codespeak.net Thu Apr 8 15:18:04 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 15:18:04 +0200 (CEST) Subject: [pypy-svn] r73547 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100408131804.C19D1282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 15:18:03 2010 New Revision: 73547 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Map macros to our functions. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Thu Apr 8 15:18:03 2010 @@ -411,6 +411,8 @@ #define PyObject_INIT_VAR(op, typeobj, size) \ ( Py_SIZE(op) = (size), PyObject_INIT((op), (typeobj)) ) +#define PyObject_INIT PyObject_Init +#define PyObject_INIT_VAR PyObject_InitVar /* #define PyObject_NEW(type, typeobj) \ ( (type *) PyObject_Init( \ From xoraxax at codespeak.net Thu Apr 8 15:21:43 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 15:21:43 +0200 (CEST) Subject: [pypy-svn] r73548 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408132143.0E0CC282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 15:21:41 2010 New Revision: 73548 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: Add __methods__ support to Py_FindMethod. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Thu Apr 8 15:21:41 2010 @@ -190,13 +190,19 @@ name = rffi.charp2str(name_ptr) methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), table) + method_list_w = [] + if methods: i = -1 while True: i = i + 1 method = methods[i] if not method.c_ml_name: break - if rffi.charp2str(method.c_ml_name) == name: # XXX expensive copying + if name == "__methods__": + method_list_w.append(space.wrap(rffi.charp2str(method.c_ml_name))) + elif rffi.charp2str(method.c_ml_name) == name: # XXX expensive copying return PyCFunction_NewEx(space, method, w_ob) + if name == "__methods__": + return space.newlist(method_list_w) raise OperationError(space.w_AttributeError, space.wrap(name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Thu Apr 8 15:21:41 2010 @@ -97,6 +97,6 @@ assert m m = re.search("xyz", "xyzxyz") assert m - assert dir(m) + assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() From afa at codespeak.net Thu Apr 8 15:44:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 15:44:11 +0200 (CEST) Subject: [pypy-svn] r73549 - in pypy/branch/cpython-extension/pypy/translator: c/gcc/test platform Message-ID: <20100408134411.0643C282BD6@codespeak.net> Author: afa Date: Thu Apr 8 15:44:10 2010 New Revision: 73549 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/cpython-extension/pypy/translator/platform/posix.py Log: Fix generation of Makefile with the --shared option Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Apr 8 15:44:10 2010 @@ -53,7 +53,12 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') + if config.translation.shared and os.name == 'posix': + env = 'LD_LIBRARY_PATH="%s" ' % (exe_name.dirpath(),) + else: + env = '' + g = os.popen( + '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) Modified: pypy/branch/cpython-extension/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/posix.py Thu Apr 8 15:44:10 2010 @@ -77,6 +77,12 @@ if shared: linkflags = self._args_for_shared(linkflags) + if shared: + libname = exe_name.new(ext='').basename + target_name = 'lib' + exe_name.new(ext=self.so_ext).basename + else: + target_name = exe_name.basename + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci @@ -98,8 +104,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -125,6 +131,16 @@ for rule in rules: m.rule(*rule) + if shared: + m.definition('SHARED_IMPORT_LIB', libname), + m.rule('main.c', '', + 'echo "' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], + '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): From jandem at codespeak.net Thu Apr 8 16:02:52 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 16:02:52 +0200 (CEST) Subject: [pypy-svn] r73550 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408140252.B8450282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 16:02:50 2010 New Revision: 73550 Added: pypy/branch/cpython-extension/pypy/module/cpyext/sliceobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add sliceobject and PySlice_GetIndicesEx Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 16:02:50 2010 @@ -55,6 +55,7 @@ import pypy.module.cpyext.pycobject import pypy.module.cpyext.sysmodule import pypy.module.cpyext.number +import pypy.module.cpyext.sliceobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 16:02:50 2010 @@ -32,6 +32,7 @@ # update these for other platforms Py_ssize_t = lltype.Signed +Py_ssize_tP = lltype.Ptr(lltype.Array(Py_ssize_t, hints={'nolength': True})) size_t = rffi.ULONG ADDR = lltype.Signed @@ -590,7 +591,8 @@ source_dir / "getargs.c", source_dir / "stringobject.c", source_dir / "mysnprintf.c", - source_dir / "pythonrun.c"], + source_dir / "pythonrun.c", + ], separate_module_sources = [code], export_symbols=export_symbols_eci, **kwds Added: pypy/branch/cpython-extension/pypy/module/cpyext/sliceobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sliceobject.py Thu Apr 8 16:02:50 2010 @@ -0,0 +1,27 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t,\ + Py_ssize_tP, build_type_checkers +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject +from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.interpreter.error import OperationError +from pypy.objspace.std.sliceobject import W_SliceObject + +PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") + + at cpython_api([PyObject, Py_ssize_t, Py_ssize_tP, Py_ssize_tP, Py_ssize_tP, + Py_ssize_tP], rffi.INT_real, error=-1) +def PySlice_GetIndicesEx(space, w_slice, length, start_p, stop_p, + step_p, slicelength_p): + """Usable replacement for PySlice_GetIndices(). Retrieve the start, + stop, and step indices from the slice object slice assuming a sequence of + length length, and store the length of the slice in slicelength. Out + of bounds indices are clipped in a manner consistent with the handling of + normal slices. + + Returns 0 on success and -1 on error with exception set.""" + if not PySlice_Check(space, w_slice): + PyErr_BadInternalCall(space) + assert isinstance(w_slice, W_SliceObject) + start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \ + w_slice.indices4(space, length) + return 0 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py Thu Apr 8 16:02:50 2010 @@ -0,0 +1,31 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP + +class TestSliceObject(BaseApiTest): + def test_slice(self, space, api): + w_i = space.wrap(10) + w_slice = space.newslice(w_i, w_i, w_i) + assert api.PySlice_Check(w_slice) + assert not api.PySlice_Check(w_i) + + def test_GetIndicesEx(self, space, api): + + w = space.wrap + + def get_indices(w_start, w_stop, w_step, length): + w_slice = space.newslice(w_start, w_stop, w_step) + values = lltype.malloc(Py_ssize_tP.TO, 4, flavor='raw') + + res = api.PySlice_GetIndicesEx(w_slice, 100, values, + rffi.ptradd(values, 1), + rffi.ptradd(values, 2), + rffi.ptradd(values, 3)) + assert res == 0 + + rv = values[0], values[1], values[2], values[3] + lltype.free(values, flavor='raw') + return rv + + assert get_indices(w(10), w(20), w(1), 200) == (10, 20, 1, 10) From jandem at codespeak.net Thu Apr 8 16:15:36 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 16:15:36 +0200 (CEST) Subject: [pypy-svn] r73551 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100408141536.88D54282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 16:15:35 2010 New Revision: 73551 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyport.h pypy/branch/cpython-extension/pypy/module/cpyext/include/sliceobject.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: add sliceobject.h, pyport.h and a few macros Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Thu Apr 8 16:15:35 2010 @@ -74,6 +74,7 @@ #include "patchlevel.h" #include "object.h" +#include "pyport.h" #include #include @@ -100,6 +101,8 @@ #include "eval.h" #include "pymem.h" #include "pycobject.h" +#include "bufferobject.h" +#include "sliceobject.h" // XXX This shouldn't be included here #include "structmember.h" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Thu Apr 8 16:15:35 2010 @@ -388,7 +388,9 @@ #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL +#define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) +#define _Py_NewReference(op) (Py_REFCNT(op) = 1) /* objimpl.h ----------------------------------------------*/ #define PyObject_DEL PyObject_Del Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyport.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pyport.h Thu Apr 8 16:15:35 2010 @@ -0,0 +1,19 @@ +#ifndef Py_PYPORT_H +#define Py_PYPORT_H + +#ifdef HAVE_STDINT_H +#include +#endif + +/* Largest possible value of size_t. + SIZE_MAX is part of C99, so it might be defined on some + platforms. If it is not defined, (size_t)-1 is a portable + definition for C89, due to the way signed->unsigned + conversion is defined. */ +#ifdef SIZE_MAX +#define PY_SIZE_MAX SIZE_MAX +#else +#define PY_SIZE_MAX ((size_t)-1) +#endif + +#endif /* Py_PYPORT_H */ Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/sliceobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/sliceobject.h Thu Apr 8 16:15:35 2010 @@ -0,0 +1,20 @@ +#ifndef Py_SLICEOBJECT_H +#define Py_SLICEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +/* The unique ellipsis object "..." */ + +PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ + +#define Py_Ellipsis (&_Py_EllipsisObject) + +typedef struct { + PyObject_HEAD +} PySliceObject; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_SLICEOBJECT_H */ From jandem at codespeak.net Thu Apr 8 16:32:17 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 16:32:17 +0200 (CEST) Subject: [pypy-svn] r73552 - in pypy/branch/cpython-extension/pypy/module/cpyext: include test Message-ID: <20100408143217.32E69282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 16:32:15 2010 New Revision: 73552 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py Log: Remove duplicate macros, add PyMem_Malloc Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Thu Apr 8 16:32:15 2010 @@ -390,8 +390,6 @@ #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) -#define _Py_NewReference(op) (Py_REFCNT(op) = 1) - /* objimpl.h ----------------------------------------------*/ #define PyObject_DEL PyObject_Del #define PyObject_New(type, typeobj) \ @@ -408,11 +406,6 @@ ) & ~(SIZEOF_VOID_P - 1) \ ) -#define PyObject_INIT(op, typeobj) \ - ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) -#define PyObject_INIT_VAR(op, typeobj, size) \ - ( Py_SIZE(op) = (size), PyObject_INIT((op), (typeobj)) ) - #define PyObject_INIT PyObject_Init #define PyObject_INIT_VAR PyObject_InitVar /* Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Thu Apr 8 16:32:15 2010 @@ -10,3 +10,5 @@ //#define PyObject_REALLOC PyMem_REALLOC #define PyObject_FREE PyMem_FREE +#define PyMem_Malloc PyMem_MALLOC +#define PyMem_Free PyMem_FREE Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py Thu Apr 8 16:32:15 2010 @@ -11,9 +11,7 @@ assert not api.PySlice_Check(w_i) def test_GetIndicesEx(self, space, api): - w = space.wrap - def get_indices(w_start, w_stop, w_step, length): w_slice = space.newslice(w_start, w_stop, w_step) values = lltype.malloc(Py_ssize_tP.TO, 4, flavor='raw') @@ -23,9 +21,7 @@ rffi.ptradd(values, 2), rffi.ptradd(values, 3)) assert res == 0 - rv = values[0], values[1], values[2], values[3] lltype.free(values, flavor='raw') return rv - assert get_indices(w(10), w(20), w(1), 200) == (10, 20, 1, 10) From xoraxax at codespeak.net Thu Apr 8 17:15:31 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 17:15:31 +0200 (CEST) Subject: [pypy-svn] r73553 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100408151531.AA05D282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 17:15:30 2010 New Revision: 73553 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Call the startup function of the cpyext module in tests. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py Thu Apr 8 17:15:30 2010 @@ -16,6 +16,7 @@ class BaseApiTest: def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext']) + cls.space.getbuiltinmodule("cpyext") class CAPI: def __getattr__(self, name): return getattr(cls.space, name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Thu Apr 8 17:15:30 2010 @@ -120,6 +120,7 @@ class AppTestCpythonExtensionBase: def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext']) + cls.space.getbuiltinmodule("cpyext") def import_module(self, name, init=None, body=''): if init is not None: From jandem at codespeak.net Thu Apr 8 17:34:15 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:34:15 +0200 (CEST) Subject: [pypy-svn] r73554 - in pypy/branch/cpython-extension/pypy/module/cpyext: include src Message-ID: <20100408153415.26F44282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:34:13 2010 New Revision: 73554 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Log: add bufferobject from CPython Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Thu Apr 8 17:34:13 2010 @@ -0,0 +1,34 @@ + +/* Buffer object interface */ + +/* Note: the object's structure is private */ + +#ifndef Py_BUFFEROBJECT_H +#define Py_BUFFEROBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +static PyTypeObject PyBuffer_Type; + +#define PyBuffer_Check(op) (Py_TYPE(op) == &PyBuffer_Type) + +#define Py_END_OF_BUFFER (-1) + +PyObject* PyBuffer_FromObject(PyObject *base, + Py_ssize_t offset, Py_ssize_t size); +PyObject* PyBuffer_FromReadWriteObject(PyObject *base, + Py_ssize_t offset, + Py_ssize_t size); + +PyObject* PyBuffer_FromMemory(void *ptr, Py_ssize_t size); +PyObject* PyBuffer_FromReadWriteMemory(void *ptr, Py_ssize_t size); + +PyObject* PyBuffer_New(Py_ssize_t size); + +void init_bufferobject(void); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_BUFFEROBJECT_H */ Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Thu Apr 8 17:34:13 2010 @@ -0,0 +1,866 @@ + +/* Buffer object implementation */ + +#include "Python.h" + + +typedef struct { + PyObject_HEAD + PyObject *b_base; + void *b_ptr; + Py_ssize_t b_size; + Py_ssize_t b_offset; + int b_readonly; + long b_hash; +} PyBufferObject; + + +enum buffer_t { + READ_BUFFER, + WRITE_BUFFER, + CHAR_BUFFER, + ANY_BUFFER +}; + +static int +get_buf(PyBufferObject *self, void **ptr, Py_ssize_t *size, + enum buffer_t buffer_type) +{ + if (self->b_base == NULL) { + assert (ptr != NULL); + *ptr = self->b_ptr; + *size = self->b_size; + } + else { + Py_ssize_t count, offset; + readbufferproc proc = 0; + PyBufferProcs *bp = self->b_base->ob_type->tp_as_buffer; + if ((*bp->bf_getsegcount)(self->b_base, NULL) != 1) { + PyErr_SetString(PyExc_TypeError, + "single-segment buffer object expected"); + return 0; + } + if ((buffer_type == READ_BUFFER) || + ((buffer_type == ANY_BUFFER) && self->b_readonly)) + proc = bp->bf_getreadbuffer; + else if ((buffer_type == WRITE_BUFFER) || + (buffer_type == ANY_BUFFER)) + proc = (readbufferproc)bp->bf_getwritebuffer; + else if (buffer_type == CHAR_BUFFER) { + if (!PyType_HasFeature(self->ob_type, + Py_TPFLAGS_HAVE_GETCHARBUFFER)) { + PyErr_SetString(PyExc_TypeError, + "Py_TPFLAGS_HAVE_GETCHARBUFFER needed"); + return 0; + } + proc = (readbufferproc)bp->bf_getcharbuffer; + } + if (!proc) { + char *buffer_type_name; + switch (buffer_type) { + case READ_BUFFER: + buffer_type_name = "read"; + break; + case WRITE_BUFFER: + buffer_type_name = "write"; + break; + case CHAR_BUFFER: + buffer_type_name = "char"; + break; + default: + buffer_type_name = "no"; + break; + } + PyErr_Format(PyExc_TypeError, + "%s buffer type not available", + buffer_type_name); + return 0; + } + if ((count = (*proc)(self->b_base, 0, ptr)) < 0) + return 0; + /* apply constraints to the start/end */ + if (self->b_offset > count) + offset = count; + else + offset = self->b_offset; + *(char **)ptr = *(char **)ptr + offset; + if (self->b_size == Py_END_OF_BUFFER) + *size = count; + else + *size = self->b_size; + if (offset + *size > count) + *size = count - offset; + } + return 1; +} + + +static PyObject * +buffer_from_memory(PyObject *base, Py_ssize_t size, Py_ssize_t offset, void *ptr, + int readonly) +{ + PyBufferObject * b; + + if (size < 0 && size != Py_END_OF_BUFFER) { + PyErr_SetString(PyExc_ValueError, + "size must be zero or positive"); + return NULL; + } + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, + "offset must be zero or positive"); + return NULL; + } + + b = PyObject_NEW(PyBufferObject, &PyBuffer_Type); + if ( b == NULL ) + return NULL; + + Py_XINCREF(base); + b->b_base = base; + b->b_ptr = ptr; + b->b_size = size; + b->b_offset = offset; + b->b_readonly = readonly; + b->b_hash = -1; + + return (PyObject *) b; +} + +static PyObject * +buffer_from_object(PyObject *base, Py_ssize_t size, Py_ssize_t offset, int readonly) +{ + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, + "offset must be zero or positive"); + return NULL; + } + if ( PyBuffer_Check(base) && (((PyBufferObject *)base)->b_base) ) { + /* another buffer, refer to the base object */ + PyBufferObject *b = (PyBufferObject *)base; + if (b->b_size != Py_END_OF_BUFFER) { + Py_ssize_t base_size = b->b_size - offset; + if (base_size < 0) + base_size = 0; + if (size == Py_END_OF_BUFFER || size > base_size) + size = base_size; + } + offset += b->b_offset; + base = b->b_base; + } + return buffer_from_memory(base, size, offset, NULL, readonly); +} + + +PyObject * +PyBuffer_FromObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size) +{ + PyBufferProcs *pb = base->ob_type->tp_as_buffer; + + if ( pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_SetString(PyExc_TypeError, "buffer object expected"); + return NULL; + } + + return buffer_from_object(base, size, offset, 1); +} + +PyObject * +PyBuffer_FromReadWriteObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size) +{ + PyBufferProcs *pb = base->ob_type->tp_as_buffer; + + if ( pb == NULL || + pb->bf_getwritebuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_SetString(PyExc_TypeError, "buffer object expected"); + return NULL; + } + + return buffer_from_object(base, size, offset, 0); +} + +PyObject * +PyBuffer_FromMemory(void *ptr, Py_ssize_t size) +{ + return buffer_from_memory(NULL, size, 0, ptr, 1); +} + +PyObject * +PyBuffer_FromReadWriteMemory(void *ptr, Py_ssize_t size) +{ + return buffer_from_memory(NULL, size, 0, ptr, 0); +} + +PyObject * +PyBuffer_New(Py_ssize_t size) +{ + PyObject *o; + PyBufferObject * b; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, + "size must be zero or positive"); + return NULL; + } + if (sizeof(*b) > PY_SSIZE_T_MAX - size) { + /* unlikely */ + return PyErr_NoMemory(); + } + /* Inline PyObject_New */ + o = (PyObject *)PyObject_MALLOC(sizeof(*b) + size); + if ( o == NULL ) + return PyErr_NoMemory(); + b = (PyBufferObject *) PyObject_INIT(o, &PyBuffer_Type); + + b->b_base = NULL; + b->b_ptr = (void *)(b + 1); + b->b_size = size; + b->b_offset = 0; + b->b_readonly = 0; + b->b_hash = -1; + + return o; +} + +/* Methods */ + +static PyObject * +buffer_new(PyTypeObject *type, PyObject *args, PyObject *kw) +{ + PyObject *ob; + Py_ssize_t offset = 0; + Py_ssize_t size = Py_END_OF_BUFFER; + + /*if (PyErr_WarnPy3k("buffer() not supported in 3.x", 1) < 0) + return NULL;*/ + + if (!_PyArg_NoKeywords("buffer()", kw)) + return NULL; + + if (!PyArg_ParseTuple(args, "O|nn:buffer", &ob, &offset, &size)) + return NULL; + return PyBuffer_FromObject(ob, offset, size); +} + +PyDoc_STRVAR(buffer_doc, +"buffer(object [, offset[, size]])\n\ +\n\ +Create a new buffer object which references the given object.\n\ +The buffer will reference a slice of the target object from the\n\ +start of the object (or at the specified offset). The slice will\n\ +extend to the end of the target object (or with the specified size)."); + + +static void +buffer_dealloc(PyBufferObject *self) +{ + Py_XDECREF(self->b_base); + PyObject_DEL(self); +} + +static int +buffer_compare(PyBufferObject *self, PyBufferObject *other) +{ + void *p1, *p2; + Py_ssize_t len_self, len_other, min_len; + int cmp; + + if (!get_buf(self, &p1, &len_self, ANY_BUFFER)) + return -1; + if (!get_buf(other, &p2, &len_other, ANY_BUFFER)) + return -1; + min_len = (len_self < len_other) ? len_self : len_other; + if (min_len > 0) { + cmp = memcmp(p1, p2, min_len); + if (cmp != 0) + return cmp < 0 ? -1 : 1; + } + return (len_self < len_other) ? -1 : (len_self > len_other) ? 1 : 0; +} + +static PyObject * +buffer_repr(PyBufferObject *self) +{ + const char *status = self->b_readonly ? "read-only" : "read-write"; + + if ( self->b_base == NULL ) + return PyString_FromFormat("<%s buffer ptr %p, size %zd at %p>", + status, + self->b_ptr, + self->b_size, + self); + else + return PyString_FromFormat( + "<%s buffer for %p, size %zd, offset %zd at %p>", + status, + self->b_base, + self->b_size, + self->b_offset, + self); +} + +static long +buffer_hash(PyBufferObject *self) +{ + void *ptr; + Py_ssize_t size; + register Py_ssize_t len; + register unsigned char *p; + register long x; + + if ( self->b_hash != -1 ) + return self->b_hash; + + /* XXX potential bugs here, a readonly buffer does not imply that the + * underlying memory is immutable. b_readonly is a necessary but not + * sufficient condition for a buffer to be hashable. Perhaps it would + * be better to only allow hashing if the underlying object is known to + * be immutable (e.g. PyString_Check() is true). Another idea would + * be to call tp_hash on the underlying object and see if it raises + * an error. */ + if ( !self->b_readonly ) + { + PyErr_SetString(PyExc_TypeError, + "writable buffers are not hashable"); + return -1; + } + + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return -1; + p = (unsigned char *) ptr; + len = size; + x = *p << 7; + while (--len >= 0) + x = (1000003*x) ^ *p++; + x ^= size; + if (x == -1) + x = -2; + self->b_hash = x; + return x; +} + +static PyObject * +buffer_str(PyBufferObject *self) +{ + void *ptr; + Py_ssize_t size; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return NULL; + return PyString_FromStringAndSize((const char *)ptr, size); +} + +/* Sequence methods */ + +static Py_ssize_t +buffer_length(PyBufferObject *self) +{ + void *ptr; + Py_ssize_t size; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return -1; + return size; +} + +static PyObject * +buffer_concat(PyBufferObject *self, PyObject *other) +{ + PyBufferProcs *pb = other->ob_type->tp_as_buffer; + void *ptr1, *ptr2; + char *p; + PyObject *ob; + Py_ssize_t size, count; + + if ( pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_BadArgument(); + return NULL; + } + if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) + { + /* ### use a different exception type/message? */ + PyErr_SetString(PyExc_TypeError, + "single-segment buffer object expected"); + return NULL; + } + + if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) + return NULL; + + /* optimize special case */ + if ( size == 0 ) + { + Py_INCREF(other); + return other; + } + + if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) + return NULL; + + assert(count <= PY_SIZE_MAX - size); + + ob = PyString_FromStringAndSize(NULL, size + count); + if ( ob == NULL ) + return NULL; + p = PyString_AS_STRING(ob); + memcpy(p, ptr1, size); + memcpy(p + size, ptr2, count); + + /* there is an extra byte in the string object, so this is safe */ + p[size + count] = '\0'; + + return ob; +} + +static PyObject * +buffer_repeat(PyBufferObject *self, Py_ssize_t count) +{ + PyObject *ob; + register char *p; + void *ptr; + Py_ssize_t size; + + if ( count < 0 ) + count = 0; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return NULL; + if (count > PY_SSIZE_T_MAX / size) { + PyErr_SetString(PyExc_MemoryError, "result too large"); + return NULL; + } + ob = PyString_FromStringAndSize(NULL, size * count); + if ( ob == NULL ) + return NULL; + + p = PyString_AS_STRING(ob); + while ( count-- ) + { + memcpy(p, ptr, size); + p += size; + } + + /* there is an extra byte in the string object, so this is safe */ + *p = '\0'; + + return ob; +} + +static PyObject * +buffer_item(PyBufferObject *self, Py_ssize_t idx) +{ + void *ptr; + Py_ssize_t size; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return NULL; + if ( idx < 0 || idx >= size ) { + PyErr_SetString(PyExc_IndexError, "buffer index out of range"); + return NULL; + } + return PyString_FromStringAndSize((char *)ptr + idx, 1); +} + +static PyObject * +buffer_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right) +{ + void *ptr; + Py_ssize_t size; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return NULL; + if ( left < 0 ) + left = 0; + if ( right < 0 ) + right = 0; + if ( right > size ) + right = size; + if ( right < left ) + right = left; + return PyString_FromStringAndSize((char *)ptr + left, + right - left); +} + +static PyObject * +buffer_subscript(PyBufferObject *self, PyObject *item) +{ + void *p; + Py_ssize_t size; + + if (!get_buf(self, &p, &size, ANY_BUFFER)) + return NULL; + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += size; + return buffer_item(self, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength, cur, i; + + if (PySlice_GetIndicesEx((PySliceObject*)item, size, + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (slicelength <= 0) + return PyString_FromStringAndSize("", 0); + else if (step == 1) + return PyString_FromStringAndSize((char *)p + start, + stop - start); + else { + PyObject *result; + char *source_buf = (char *)p; + char *result_buf = (char *)PyMem_Malloc(slicelength); + + if (result_buf == NULL) + return PyErr_NoMemory(); + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + result_buf[i] = source_buf[cur]; + } + + result = PyString_FromStringAndSize(result_buf, + slicelength); + PyMem_Free(result_buf); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "sequence index must be integer"); + return NULL; + } +} + +static int +buffer_ass_item(PyBufferObject *self, Py_ssize_t idx, PyObject *other) +{ + PyBufferProcs *pb; + void *ptr1, *ptr2; + Py_ssize_t size; + Py_ssize_t count; + + if ( self->b_readonly ) { + PyErr_SetString(PyExc_TypeError, + "buffer is read-only"); + return -1; + } + + if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) + return -1; + + if (idx < 0 || idx >= size) { + PyErr_SetString(PyExc_IndexError, + "buffer assignment index out of range"); + return -1; + } + + pb = other ? other->ob_type->tp_as_buffer : NULL; + if ( pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_BadArgument(); + return -1; + } + if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) + { + /* ### use a different exception type/message? */ + PyErr_SetString(PyExc_TypeError, + "single-segment buffer object expected"); + return -1; + } + + if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) + return -1; + if ( count != 1 ) { + PyErr_SetString(PyExc_TypeError, + "right operand must be a single byte"); + return -1; + } + + ((char *)ptr1)[idx] = *(char *)ptr2; + return 0; +} + +static int +buffer_ass_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right, PyObject *other) +{ + PyBufferProcs *pb; + void *ptr1, *ptr2; + Py_ssize_t size; + Py_ssize_t slice_len; + Py_ssize_t count; + + if ( self->b_readonly ) { + PyErr_SetString(PyExc_TypeError, + "buffer is read-only"); + return -1; + } + + pb = other ? other->ob_type->tp_as_buffer : NULL; + if ( pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_BadArgument(); + return -1; + } + if ( (*pb->bf_getsegcount)(other, NULL) != 1 ) + { + /* ### use a different exception type/message? */ + PyErr_SetString(PyExc_TypeError, + "single-segment buffer object expected"); + return -1; + } + if (!get_buf(self, &ptr1, &size, ANY_BUFFER)) + return -1; + if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 ) + return -1; + + if ( left < 0 ) + left = 0; + else if ( left > size ) + left = size; + if ( right < left ) + right = left; + else if ( right > size ) + right = size; + slice_len = right - left; + + if ( count != slice_len ) { + PyErr_SetString( + PyExc_TypeError, + "right operand length must match slice length"); + return -1; + } + + if ( slice_len ) + memcpy((char *)ptr1 + left, ptr2, slice_len); + + return 0; +} + +static int +buffer_ass_subscript(PyBufferObject *self, PyObject *item, PyObject *value) +{ + PyBufferProcs *pb; + void *ptr1, *ptr2; + Py_ssize_t selfsize; + Py_ssize_t othersize; + + if ( self->b_readonly ) { + PyErr_SetString(PyExc_TypeError, + "buffer is read-only"); + return -1; + } + + pb = value ? value->ob_type->tp_as_buffer : NULL; + if ( pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL ) + { + PyErr_BadArgument(); + return -1; + } + if ( (*pb->bf_getsegcount)(value, NULL) != 1 ) + { + /* ### use a different exception type/message? */ + PyErr_SetString(PyExc_TypeError, + "single-segment buffer object expected"); + return -1; + } + if (!get_buf(self, &ptr1, &selfsize, ANY_BUFFER)) + return -1; + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += selfsize; + return buffer_ass_item(self, i, value); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength; + + if (PySlice_GetIndicesEx((PySliceObject *)item, selfsize, + &start, &stop, &step, &slicelength) < 0) + return -1; + + if ((othersize = (*pb->bf_getreadbuffer)(value, 0, &ptr2)) < 0) + return -1; + + if (othersize != slicelength) { + PyErr_SetString( + PyExc_TypeError, + "right operand length must match slice length"); + return -1; + } + + if (slicelength == 0) + return 0; + else if (step == 1) { + memcpy((char *)ptr1 + start, ptr2, slicelength); + return 0; + } + else { + Py_ssize_t cur, i; + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + ((char *)ptr1)[cur] = ((char *)ptr2)[i]; + } + + return 0; + } + } else { + PyErr_SetString(PyExc_TypeError, + "buffer indices must be integers"); + return -1; + } +} + +/* Buffer methods */ + +static Py_ssize_t +buffer_getreadbuf(PyBufferObject *self, Py_ssize_t idx, void **pp) +{ + Py_ssize_t size; + if ( idx != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent buffer segment"); + return -1; + } + if (!get_buf(self, pp, &size, READ_BUFFER)) + return -1; + return size; +} + +static Py_ssize_t +buffer_getwritebuf(PyBufferObject *self, Py_ssize_t idx, void **pp) +{ + Py_ssize_t size; + + if ( self->b_readonly ) + { + PyErr_SetString(PyExc_TypeError, "buffer is read-only"); + return -1; + } + + if ( idx != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent buffer segment"); + return -1; + } + if (!get_buf(self, pp, &size, WRITE_BUFFER)) + return -1; + return size; +} + +static Py_ssize_t +buffer_getsegcount(PyBufferObject *self, Py_ssize_t *lenp) +{ + void *ptr; + Py_ssize_t size; + if (!get_buf(self, &ptr, &size, ANY_BUFFER)) + return -1; + if (lenp) + *lenp = size; + return 1; +} + +static Py_ssize_t +buffer_getcharbuf(PyBufferObject *self, Py_ssize_t idx, const char **pp) +{ + void *ptr; + Py_ssize_t size; + if ( idx != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent buffer segment"); + return -1; + } + if (!get_buf(self, &ptr, &size, CHAR_BUFFER)) + return -1; + *pp = (const char *)ptr; + return size; +} + +void init_bufferobject(void) +{ + PyType_Ready(&PyBuffer_Type); +} + +static PySequenceMethods buffer_as_sequence = { + (lenfunc)buffer_length, /*sq_length*/ + (binaryfunc)buffer_concat, /*sq_concat*/ + (ssizeargfunc)buffer_repeat, /*sq_repeat*/ + (ssizeargfunc)buffer_item, /*sq_item*/ + (ssizessizeargfunc)buffer_slice, /*sq_slice*/ + (ssizeobjargproc)buffer_ass_item, /*sq_ass_item*/ + (ssizessizeobjargproc)buffer_ass_slice, /*sq_ass_slice*/ +}; + +static PyMappingMethods buffer_as_mapping = { + (lenfunc)buffer_length, + (binaryfunc)buffer_subscript, + (objobjargproc)buffer_ass_subscript, +}; + +static PyBufferProcs buffer_as_buffer = { + (readbufferproc)buffer_getreadbuf, + (writebufferproc)buffer_getwritebuf, + (segcountproc)buffer_getsegcount, + (charbufferproc)buffer_getcharbuf, +}; + +static PyTypeObject PyBuffer_Type = { + PyObject_HEAD_INIT(NULL) + 0, + "buffer", + sizeof(PyBufferObject), + 0, + (destructor)buffer_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + (cmpfunc)buffer_compare, /* tp_compare */ + (reprfunc)buffer_repr, /* tp_repr */ + 0, /* tp_as_number */ + &buffer_as_sequence, /* tp_as_sequence */ + &buffer_as_mapping, /* tp_as_mapping */ + (hashfunc)buffer_hash, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)buffer_str, /* tp_str */ + 0, //XXX PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &buffer_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GETCHARBUFFER, /* tp_flags */ + buffer_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + buffer_new, /* tp_new */ +}; + From jandem at codespeak.net Thu Apr 8 17:35:30 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:35:30 +0200 (CEST) Subject: [pypy-svn] r73555 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408153530.2EF90282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:35:28 2010 New Revision: 73555 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: call init functions on startup Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 17:35:28 2010 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.module.cpyext.state import State from pypy.module.cpyext import api - +from pypy.rpython.lltypesystem import rffi, lltype class Module(MixedModule): interpleveldefs = { @@ -30,6 +30,9 @@ space.wrap(state.api_lib)) else: state.init_r2w_from_w2r() + + for func in api.INIT_FUNCTIONS: + func() # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 17:35:28 2010 @@ -229,6 +229,7 @@ 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', + 'PyBuffer_FromMemory', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -236,6 +237,7 @@ '_Py_TrueStruct#': ('PyObject*', 'space.w_True'), '_Py_ZeroStruct#': ('PyObject*', 'space.w_False'), } +INIT_FUNCTIONS = [] for exc_name in exceptions.Module.interpleveldefs.keys(): GLOBALS['PyExc_' + exc_name] = ('PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) @@ -498,7 +500,10 @@ ctypes.c_void_p) setup_va_functions(eci) - + + INIT_FUNCTIONS.extend([ + rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) + ]) return modulename.new(ext='') def generate_macros(export_symbols, rename=True, do_deref=True): @@ -592,6 +597,7 @@ source_dir / "stringobject.c", source_dir / "mysnprintf.c", source_dir / "pythonrun.c", + source_dir / "bufferobject.c" ], separate_module_sources = [code], export_symbols=export_symbols_eci, From jandem at codespeak.net Thu Apr 8 17:36:45 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:36:45 +0200 (CEST) Subject: [pypy-svn] r73556 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100408153645.B4A07282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:36:44 2010 New Revision: 73556 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py Log: add tests for bufferobject Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py Thu Apr 8 17:36:44 2010 @@ -0,0 +1,29 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.api import PyObject +from pypy.module.cpyext.pyobject import Py_DecRef + +class AppTestBufferObject(AppTestCpythonExtensionBase): + def test_FromMemory(self): + module = self.import_extension('foo', [ + ("get_FromMemory", "METH_NOARGS", + """ + cbuf = malloc(4); + cbuf[0] = 'a'; + cbuf[1] = 'b'; + cbuf[2] = 'c'; + cbuf[3] = '\0'; + return PyBuffer_FromMemory(cbuf, 4); + """), + ("free_buffer", "METH_NOARGS", + """ + free(cbuf); + Py_RETURN_NONE; + """) + ], prologue = """ + char* cbuf = NULL; + """) + w_buffer = module.get_FromMemory() + assert False, w_buffer + module.free_buffer() From jandem at codespeak.net Thu Apr 8 17:39:24 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:39:24 +0200 (CEST) Subject: [pypy-svn] r73557 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408153924.3774D282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:39:22 2010 New Revision: 73557 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py Log: Add PyNumber_AsSsize_t, not finished but enough to test bufferobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Thu Apr 8 17:39:22 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t from pypy.module.cpyext.pyobject import PyObject from pypy.rpython.lltypesystem import rffi, lltype @@ -13,3 +13,15 @@ return 1 except OperationError: return 0 + + at cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) +def PyNumber_AsSsize_t(space, w_obj, w_exc): + """Returns o converted to a Py_ssize_t value if o can be interpreted as an + integer. If o can be converted to a Python int or long but the attempt to + convert to a Py_ssize_t value would raise an OverflowError, then the + exc argument is the type of exception that will be raised (usually + IndexError or OverflowError). If exc is NULL, then the + exception is cleared and the value is clipped to PY_SSIZE_T_MIN for a negative + integer or PY_SSIZE_T_MAX for a positive integer. + """ + return space.int_w(w_obj) #XXX: this is wrong From jandem at codespeak.net Thu Apr 8 17:40:25 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:40:25 +0200 (CEST) Subject: [pypy-svn] r73558 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408154025.50E96282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:40:23 2010 New Revision: 73558 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Hack in typeobject.py to load bufferobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 8 17:40:23 2010 @@ -208,8 +208,11 @@ convert_member_defs(space, dict_w, pto.c_tp_members, pto) full_name = rffi.charp2str(pto.c_tp_name) - module_name, extension_name = rsplit(full_name, ".", 1) - dict_w["__module__"] = space.wrap(module_name) + if '.' in full_name: + module_name, extension_name = rsplit(full_name, ".", 1) + dict_w["__module__"] = space.wrap(module_name) + else: + extension_name = None W_TypeObject.__init__(self, space, extension_name, bases_w or [space.w_object], dict_w) From jandem at codespeak.net Thu Apr 8 17:42:44 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Thu, 8 Apr 2010 17:42:44 +0200 (CEST) Subject: [pypy-svn] r73559 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408154244.10BB1282BD6@codespeak.net> Author: jandem Date: Thu Apr 8 17:42:43 2010 New Revision: 73559 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: comment out offending lines Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 17:42:43 2010 @@ -31,8 +31,9 @@ else: state.init_r2w_from_w2r() - for func in api.INIT_FUNCTIONS: - func() + #XXX: segfaults + #for func in api.INIT_FUNCTIONS: + # func() # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject From trundle at codespeak.net Thu Apr 8 17:52:39 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 8 Apr 2010 17:52:39 +0200 (CEST) Subject: [pypy-svn] r73560 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408155239.240E1282BD6@codespeak.net> Author: trundle Date: Thu Apr 8 17:52:36 2010 New Revision: 73560 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_GetDefaultEncoding and PyUnicode_SetDefaultEncoding. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Thu Apr 8 17:52:36 2010 @@ -9,6 +9,24 @@ unichar = rffi.sizeof(Py_UNICODE) assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 4 * unichar + encoding = rffi.charp2str(api.PyUnicode_GetDefaultEncoding()) + w_default_encoding = space.call_function( + space.sys.get('getdefaultencoding') + ) + assert encoding == space.unwrap(w_default_encoding) + invalid = rffi.str2charp('invalid') + utf_8 = rffi.str2charp('utf-8') + prev_encoding = rffi.str2charp(space.unwrap(w_default_encoding)) + assert api.PyUnicode_SetDefaultEncoding(invalid) == -1 + assert api.PyErr_Occurred() is space.w_LookupError + api.PyErr_Clear() + assert api.PyUnicode_SetDefaultEncoding(utf_8) == 0 + assert rffi.charp2str(api.PyUnicode_GetDefaultEncoding()) == 'utf-8' + assert api.PyUnicode_SetDefaultEncoding(prev_encoding) == 0 + rffi.free_charp(invalid) + rffi.free_charp(utf_8) + rffi.free_charp(prev_encoding) + def test_AS(self, space, api): word = space.wrap(u'spam') array = rffi.cast(rffi.CWCHARP, api.PyUnicode_AS_DATA(word)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Thu Apr 8 17:52:36 2010 @@ -4,7 +4,13 @@ from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, build_type_checkers, cpython_api) from pypy.module.cpyext.pyobject import PyObject, from_ref -from pypy.objspace.std import unicodeobject +from pypy.module.sys.interp_encoding import setdefaultencoding +from pypy.objspace.std import unicodeobject, unicodetype + +# Buffer for the default encoding (used by PyUnicde_GetDefaultEncoding) +DEFAULT_ENCODING_SIZE = 100 +default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE, + flavor='raw', zero=True) PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode") @@ -102,3 +108,23 @@ raise OperationError(space.w_TypeError, space.wrap("expected unicode object")) return PyUnicode_AS_UNICODE(space, ref) + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def PyUnicode_GetDefaultEncoding(space): + """Returns the currently active default encoding.""" + if default_encoding[0] == '\x00': + encoding = unicodetype.getdefaultencoding(space) + i = 0 + while i < len(encoding) and i < DEFAULT_ENCODING_SIZE: + default_encoding[i] = encoding[i] + i += 1 + return default_encoding + + at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1) +def PyUnicode_SetDefaultEncoding(space, encoding): + """Sets the currently active default encoding. Returns 0 on + success, -1 in case of an error.""" + w_encoding = space.wrap(rffi.charp2str(encoding)) + setdefaultencoding(space, w_encoding) + default_encoding[0] = '\x00' + return 0 From getxsick at codespeak.net Thu Apr 8 18:06:10 2010 From: getxsick at codespeak.net (getxsick at codespeak.net) Date: Thu, 8 Apr 2010 18:06:10 +0200 (CEST) Subject: [pypy-svn] r73561 - pypy/branch/decouple-host-opcodes/pypy/interpreter/test Message-ID: <20100408160610.6342936C225@codespeak.net> Author: getxsick Date: Thu Apr 8 18:06:09 2010 New Revision: 73561 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py Log: Remove redundant import Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_code.py Thu Apr 8 18:06:09 2010 @@ -1,6 +1,5 @@ from pypy.conftest import gettestobjspace from pypy.interpreter import gateway -from pypy.interpreter import baseobjspace import py class AppTestCodeIntrospection: From xoraxax at codespeak.net Thu Apr 8 18:35:00 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 18:35:00 +0200 (CEST) Subject: [pypy-svn] r73563 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src Message-ID: <20100408163500.3BD59282BDA@codespeak.net> Author: xoraxax Date: Thu Apr 8 18:34:58 2010 New Revision: 73563 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Log: Refactored code a bit but we still get the segfault. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 18:34:58 2010 @@ -34,6 +34,7 @@ #XXX: segfaults #for func in api.INIT_FUNCTIONS: # func() + # state.check_and_raise_exception() # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 18:34:58 2010 @@ -28,7 +28,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem.lloperation import llop -DEBUG_WRAPPER = False +DEBUG_WRAPPER = True # update these for other platforms Py_ssize_t = lltype.Signed @@ -229,7 +229,7 @@ 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', - 'PyBuffer_FromMemory', + 'PyBuffer_FromMemory', 'PyBufferType', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -407,6 +407,10 @@ TP, compilation_info=eci) globals()['va_get_%s' % name_no_star] = func +def setup_init_functions(eci): + INIT_FUNCTIONS.extend([ + rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) + ]) def bootstrap_types(space): from pypy.module.cpyext.pyobject import make_ref @@ -500,10 +504,8 @@ ctypes.c_void_p) setup_va_functions(eci) - - INIT_FUNCTIONS.extend([ - rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) - ]) + + setup_init_functions(eci) return modulename.new(ext='') def generate_macros(export_symbols, rename=True, do_deref=True): @@ -640,6 +642,7 @@ for name, func in FUNCTIONS_STATIC.iteritems(): func.get_wrapper(space).c_name = name + setup_init_functions(eci) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Thu Apr 8 18:34:58 2010 @@ -9,7 +9,7 @@ extern "C" { #endif -static PyTypeObject PyBuffer_Type; +PyTypeObject PyBuffer_Type; #define PyBuffer_Check(op) (Py_TYPE(op) == &PyBuffer_Type) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Thu Apr 8 18:34:58 2010 @@ -2,6 +2,9 @@ /* Buffer object implementation */ #include "Python.h" +#include +#include +#include typedef struct { @@ -822,7 +825,7 @@ (charbufferproc)buffer_getcharbuf, }; -static PyTypeObject PyBuffer_Type = { +PyTypeObject PyBuffer_Type = { PyObject_HEAD_INIT(NULL) 0, "buffer", From xoraxax at codespeak.net Thu Apr 8 18:52:34 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 18:52:34 +0200 (CEST) Subject: [pypy-svn] r73564 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100408165234.6F01F282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 18:52:33 2010 New Revision: 73564 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Log: Fix linking error on windows. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Thu Apr 8 18:52:33 2010 @@ -9,7 +9,7 @@ extern "C" { #endif -PyTypeObject PyBuffer_Type; +PyAPI_DATA(PyTypeObject) PyBuffer_Type; #define PyBuffer_Check(op) (Py_TYPE(op) == &PyBuffer_Type) From xoraxax at codespeak.net Thu Apr 8 18:56:12 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 18:56:12 +0200 (CEST) Subject: [pypy-svn] r73565 - pypy/branch/cpython-extension/pypy/module/cpyext/src Message-ID: <20100408165612.E8310282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 18:56:11 2010 New Revision: 73565 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Log: Remove includes. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Thu Apr 8 18:56:11 2010 @@ -2,9 +2,6 @@ /* Buffer object implementation */ #include "Python.h" -#include -#include -#include typedef struct { From xoraxax at codespeak.net Thu Apr 8 18:57:22 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 18:57:22 +0200 (CEST) Subject: [pypy-svn] r73566 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408165722.34E29282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 18:57:20 2010 New Revision: 73566 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add underscore. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 18:57:20 2010 @@ -229,7 +229,7 @@ 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', - 'PyBuffer_FromMemory', 'PyBufferType', + 'PyBuffer_FromMemory', 'PyBuffer_Type', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur From trundle at codespeak.net Thu Apr 8 19:14:19 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Thu, 8 Apr 2010 19:14:19 +0200 (CEST) Subject: [pypy-svn] r73567 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408171419.8E1EC282BD6@codespeak.net> Author: trundle Date: Thu Apr 8 19:14:18 2010 New Revision: 73567 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_AsEncodedString. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Thu Apr 8 19:14:18 2010 @@ -5780,15 +5780,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) -def PyUnicode_AsEncodedString(space, unicode, encoding, errors): - """Encode a Unicode object and return the result as Python string object. - encoding and errors have the same meaning as the parameters of the same name - in the Unicode encode() method. The codec to be used is looked up using - the Python codec registry. Return NULL if an exception was raised by the - codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_DecodeUTF8(space, s, size, errors): """Create a Unicode object by decoding size bytes of the UTF-8 encoded string Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Thu Apr 8 19:14:18 2010 @@ -39,6 +39,14 @@ raises(TypeError, api.PyUnicode_AsUnicode(space.wrap('spam'))) api.PyErr_Clear() + utf_8 = rffi.str2charp('utf-8') + encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp?m'), + utf_8, None) + assert space.unwrap(encoded) == 'sp\xc3\xa4m' + raises(TypeError, api.PyUnicode_AsEncodedString, + space.wrap(''), None, None) + rffi.free_charp(utf_8) + def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Thu Apr 8 19:14:18 2010 @@ -3,7 +3,9 @@ from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, build_type_checkers, cpython_api) +from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import PyObject, from_ref +from pypy.module.cpyext.stringobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype @@ -128,3 +130,20 @@ setdefaultencoding(space, w_encoding) default_encoding[0] = '\x00' return 0 + + at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) +def PyUnicode_AsEncodedString(space, w_unicode, encoding, errors): + """Encode a Unicode object and return the result as Python string object. + encoding and errors have the same meaning as the parameters of the same name + in the Unicode encode() method. The codec to be used is looked up using + the Python codec registry. Return NULL if an exception was raised by the + codec.""" + if not PyUnicode_Check(space, w_unicode): + PyErr_BadArgument() + + w_encoding = w_errors = None + if encoding: + w_encoding = rffi.charp2str(encoding) + if errors: + w_errors = rffi.charp2str(encoding) + return unicodetype.encode_object(space, w_unicode, w_encoding, w_errors) From fijal at codespeak.net Thu Apr 8 20:36:56 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Apr 2010 20:36:56 +0200 (CEST) Subject: [pypy-svn] r73568 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408183656.B047C282BD6@codespeak.net> Author: fijal Date: Thu Apr 8 20:36:54 2010 New Revision: 73568 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Amend todo Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Thu Apr 8 20:36:54 2010 @@ -20,4 +20,9 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. - - Test cpyext together with the jit. + - Test cpyext together with the jit. Branch does not work, but it's not + cpyext's fault (it doesn't work even with --withoutmod-cpyext). + + - sort out pypy's buffer protocol. PyPy's buffer right now don't support + raw memory (except array which supports it in a hackish way), which + should be fixed in order to make it nicely work with cpyext. \ No newline at end of file From fijal at codespeak.net Thu Apr 8 20:38:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Apr 2010 20:38:35 +0200 (CEST) Subject: [pypy-svn] r73569 - pypy/extradoc/talk/oopsla2010 Message-ID: <20100408183835.F3A71282BD6@codespeak.net> Author: fijal Date: Thu Apr 8 20:38:34 2010 New Revision: 73569 Modified: pypy/extradoc/talk/oopsla2010/paper.txt Log: Amend example a bit Modified: pypy/extradoc/talk/oopsla2010/paper.txt ============================================================================== --- pypy/extradoc/talk/oopsla2010/paper.txt (original) +++ pypy/extradoc/talk/oopsla2010/paper.txt Thu Apr 8 20:38:34 2010 @@ -40,6 +40,10 @@ > return i / j E ZeroDivisionError: integer division or modulo by zero +In this example we can see that py.test (after the test failure) looked up +the traceback for introspecting where in source the error has happened. +It also printed out contents of local variables, which greatly improve +the visibility of an error. In this paper we present the way we deal with frames in order to completely (or mostly) avoid costs associated with them. Our approach is giving @@ -78,7 +82,13 @@ Related work ============ +XXX + Technical background ==================== +To illustrate problem from the simplest possible perspective, we present a +small bytecode-based language with frame introspection capabilities. The +language itself is not of any use, but provides very similar problems +to those encountered on a larger scale. From fijal at codespeak.net Thu Apr 8 20:54:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Apr 2010 20:54:20 +0200 (CEST) Subject: [pypy-svn] r73570 - pypy/trunk/pypy/jit/tl Message-ID: <20100408185420.AF494282BD6@codespeak.net> Author: fijal Date: Thu Apr 8 20:54:19 2010 New Revision: 73570 Added: pypy/trunk/pypy/jit/tl/tinyframe.py (contents, props changed) Log: A minimal interpreter description for calling, inlining and frame introspection Added: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Thu Apr 8 20:54:19 2010 @@ -0,0 +1,28 @@ + +""" Interpreter for a tiny interpreter with frame introspection. Supports +integer values and function values. The machine is +register based with untyped registers. + +Opcodes: +ADD r1, r2 => r3 # integer addition or function combination, + # depending on argument types + # if r1 has a function f and r2 has a function g + # the result will be a function lambda arg : f(g(arg)) +INTROSPECT r1 => r2 # frame introspection - load a register with number + # pointed by r1 (must be int) to r2 +PRINT r # print a register +CALL r1 r2 # call a function in register one with argument in r2 +LOAD => r # load a function named name into register r +LOAD => r # load an integer constant into register r + +function argument always comes in r0 +""" + +class Code(object): + _immutable_ = True + + def __init__(self, code): + code = code + +def compile(strrepr): + pass From afa at codespeak.net Thu Apr 8 22:29:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 22:29:22 +0200 (CEST) Subject: [pypy-svn] r73571 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100408202922.4F86F282BD6@codespeak.net> Author: afa Date: Thu Apr 8 22:29:19 2010 New Revision: 73571 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py Log: allow cpyext to call C init_xxx() functions on startup. Fix test_bufferobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 22:29:19 2010 @@ -30,11 +30,10 @@ space.wrap(state.api_lib)) else: state.init_r2w_from_w2r() - - #XXX: segfaults - #for func in api.INIT_FUNCTIONS: - # func() - # state.check_and_raise_exception() + + for func in api.INIT_FUNCTIONS: + func() + state.check_and_raise_exception() # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 8 22:29:19 2010 @@ -229,7 +229,7 @@ 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', - 'PyBuffer_FromMemory', 'PyBuffer_Type', + 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -474,11 +474,9 @@ '\n'.join(functions)) eci = build_eci(True, export_symbols, code) - eci = eci.convert_sources_to_files() - modulename = platform.platform.compile( - [], eci, - outputfilename=str(udir / "module_cache" / "pypyapi"), - standalone=False) + eci = eci.compile_shared_lib( + outputfilename=str(udir / "module_cache" / "pypyapi")) + modulename = py.path.local(eci.libraries[-1]) bootstrap_types(space) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py Thu Apr 8 22:29:19 2010 @@ -13,7 +13,7 @@ cbuf[0] = 'a'; cbuf[1] = 'b'; cbuf[2] = 'c'; - cbuf[3] = '\0'; + cbuf[3] = '\\0'; return PyBuffer_FromMemory(cbuf, 4); """), ("free_buffer", "METH_NOARGS", @@ -22,8 +22,8 @@ Py_RETURN_NONE; """) ], prologue = """ - char* cbuf = NULL; + static char* cbuf = NULL; """) - w_buffer = module.get_FromMemory() - assert False, w_buffer + buffer = module.get_FromMemory() + assert str(buffer) == 'abc\0' module.free_buffer() From afa at codespeak.net Thu Apr 8 22:49:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Apr 2010 22:49:53 +0200 (CEST) Subject: [pypy-svn] r73572 - pypy/branch/cpython-extension/pypy/translator/tool Message-ID: <20100408204953.156B9282BD6@codespeak.net> Author: afa Date: Thu Apr 8 22:49:51 2010 New Revision: 73572 Modified: pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py Log: This belongs to the previous commit Modified: pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py Thu Apr 8 22:49:51 2010 @@ -295,18 +295,21 @@ d['separate_module_files'] = () return files, ExternalCompilationInfo(**d) - def compile_shared_lib(self): + def compile_shared_lib(self, outputfilename=None): self = self.convert_sources_to_files() if not self.separate_module_files: return self - # find more or less unique name there - basepath = py.path.local(self.separate_module_files[0]).dirpath() - pth = basepath.join('externmod').new(ext=host.so_ext) - num = 0 - while pth.check(): - pth = basepath.join('externmod_%d' % (num,)).new(ext=host.so_ext) - num += 1 - lib = str(host.compile([], self, outputfilename=pth.purebasename, + if outputfilename is None: + # find more or less unique name there + basepath = py.path.local(self.separate_module_files[0]).dirpath() + pth = basepath.join('externmod').new(ext=host.so_ext) + num = 0 + while pth.check(): + pth = basepath.join( + 'externmod_%d' % (num,)).new(ext=host.so_ext) + num += 1 + outputfilename=pth.purebasename + lib = str(host.compile([], self, outputfilename=outputfilename, standalone=False)) d = self._copy_attributes() d['libraries'] += (lib,) From xoraxax at codespeak.net Thu Apr 8 22:57:00 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 8 Apr 2010 22:57:00 +0200 (CEST) Subject: [pypy-svn] r73573 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100408205700.30850282BD6@codespeak.net> Author: xoraxax Date: Thu Apr 8 22:56:58 2010 New Revision: 73573 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Fix lost objects in the tests. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 8 22:56:58 2010 @@ -34,6 +34,7 @@ for func in api.INIT_FUNCTIONS: func() state.check_and_raise_exception() + state.non_heaptypes[:] = [] # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject From afa at codespeak.net Fri Apr 9 11:29:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 11:29:18 +0200 (CEST) Subject: [pypy-svn] r73576 - pypy/branch/cpython-extension/pypy/lib Message-ID: <20100409092918.D31BA282BAD@codespeak.net> Author: afa Date: Fri Apr 9 11:29:16 2010 New Revision: 73576 Modified: pypy/branch/cpython-extension/pypy/lib/identity_dict.py Log: Add copy() to identity_dict, for the pypy-c version as well Modified: pypy/branch/cpython-extension/pypy/lib/identity_dict.py ============================================================================== --- pypy/branch/cpython-extension/pypy/lib/identity_dict.py (original) +++ pypy/branch/cpython-extension/pypy/lib/identity_dict.py Fri Apr 9 11:29:16 2010 @@ -32,7 +32,7 @@ def copy(self): d = type(self)() - d.update(self.items()) + d.update(self.iteritems()) assert len(d) == len(self) return d @@ -58,6 +58,11 @@ def __contains__(self, arg): return arg in self._dict + def copy(self): + d = type(self)() + d.update(self.iteritems()) + assert len(d) == len(self) + return d if idict is None: identity_dict = IdentityDictPurePython From afa at codespeak.net Fri Apr 9 11:47:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 11:47:10 +0200 (CEST) Subject: [pypy-svn] r73577 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20100409094710.14551282BAD@codespeak.net> Author: afa Date: Fri Apr 9 11:47:09 2010 New Revision: 73577 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: When the compiler chooses to not inline a __inline function, it is still marked as PUBLIC in the assembler generated by the Microsoft compiler. Try to ignore these symbols when they come from the winsock2 compatibility header. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Fri Apr 9 11:47:09 2010 @@ -337,7 +337,7 @@ else: funcname = self.funcname while 1: - label = '__gcmap_%s__%s_%d' % (self.filetag, self.funcname, k) + label = '__gcmap_%s__%s_%d' % (self.filetag, funcname, k) if label not in self.labels: break k += 1 @@ -833,6 +833,8 @@ LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' OFFSET_LABELS = 0 + r_segmentstart = re.compile(r"[_A-Z]+\tSEGMENT$") + r_segmentend = re.compile(r"[_A-Z]+\tENDS$") r_functionstart = re.compile(r"; Function compile flags: ") r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") @@ -1012,7 +1014,8 @@ def process(self, iterlines, newfile, entrypoint='main', filename='?'): for in_function, lines in self.find_functions(iterlines): if in_function: - lines = self.process_function(lines, entrypoint, filename) + tracker = self.process_function(lines, entrypoint, filename) + lines = tracker.lines self.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') @@ -1040,25 +1043,24 @@ else: self.gcmaptable.extend(table) self.seen_main |= is_main - return tracker.lines + return tracker class ElfAssemblerParser(AssemblerParser): format = "elf" FunctionGcRootTracker = ElfFunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_function = False for line in iterlines: - if cls.FunctionGcRootTracker.r_functionstart.match(line): + if self.FunctionGcRootTracker.r_functionstart.match(line): assert not in_function, ( "missed the end of the previous function") yield False, functionlines in_function = True functionlines = [] functionlines.append(line) - if cls.FunctionGcRootTracker.r_functionend.match(line): + if self.FunctionGcRootTracker.r_functionend.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -1088,22 +1090,21 @@ ] r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_text = False in_function = False for n, line in enumerate(iterlines): - if cls.r_textstart.match(line): + if self.r_textstart.match(line): assert not in_text, "unexpected repeated .text start: %d" % n in_text = True - elif cls.r_sectionstart.match(line): + elif self.r_sectionstart.match(line): if in_function: yield in_function, functionlines functionlines = [] in_text = False in_function = False - elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -1121,17 +1122,16 @@ format = "mingw32" FunctionGcRootTracker = Mingw32FunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_text = False in_function = False for n, line in enumerate(iterlines): - if cls.r_textstart.match(line): + if self.r_textstart.match(line): in_text = True - elif cls.r_sectionstart.match(line): + elif self.r_sectionstart.match(line): in_text = False - elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -1143,19 +1143,41 @@ format = "msvc" FunctionGcRootTracker = MsvcFunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_function = False + in_segment = False + ignore_public = False + self.inline_functions = {} for line in iterlines: - if cls.FunctionGcRootTracker.r_functionstart.match(line): + if line.startswith('; File '): + filename = line[:-1].split(' ', 2)[2] + ignore_public = ('wspiapi.h' in filename.lower()) + if ignore_public: + # this header define __inline functions, that are + # still marked as PUBLIC in the generated assembler + if line.startswith(';\tCOMDAT '): + funcname = line[:-1].split(' ', 1)[1] + self.inline_functions[funcname] = True + elif line.startswith('PUBLIC\t'): + funcname = line[:-1].split('\t')[1] + self.inline_functions[funcname] = True + + if self.FunctionGcRootTracker.r_segmentstart.match(line): + in_segment = True + elif self.FunctionGcRootTracker.r_functionstart.match(line): assert not in_function, ( "missed the end of the previous function") - yield False, functionlines in_function = True - functionlines = [] + if in_segment: + yield False, functionlines + functionlines = [] functionlines.append(line) - if cls.FunctionGcRootTracker.r_functionend.match(line): + if self.FunctionGcRootTracker.r_segmentend.match(line): + yield False, functionlines + in_segment = False + functionlines = [] + elif self.FunctionGcRootTracker.r_functionend.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -1181,14 +1203,18 @@ # compiler: every string or float constant is exported # with a name built after its value, and will conflict # with other modules. - if line.startswith("PUBLIC\t__real@"): - line = '; ' + line - if line.startswith("PUBLIC\t__mask@@"): - line = '; ' + line - elif line.startswith("PUBLIC\t??_C@"): - line = '; ' + line - elif line == "PUBLIC\t__$ArrayPad$\n": - line = '; ' + line + if line.startswith("PUBLIC\t"): + symbol = line[:-1].split()[1] + if symbol.startswith('__real@'): + line = '; ' + line + elif symbol.startswith("__mask@@"): + line = '; ' + line + elif symbol.startswith("??_C@"): + line = '; ' + line + elif symbol == "__$ArrayPad$": + line = '; ' + line + elif symbol in self.inline_functions: + line = '; ' + line # The msvc compiler writes "fucomip ST(1)" when the correct # syntax is "fucomip ST, ST(1)" @@ -1451,7 +1477,8 @@ parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) for in_function, lines in parser.find_functions(iterlines): if in_function: - lines = parser.process_function(lines, entrypoint, filename) + tracker = parser.process_function(lines, entrypoint, filename) + lines = tracker.lines parser.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') From afa at codespeak.net Fri Apr 9 13:31:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 13:31:57 +0200 (CEST) Subject: [pypy-svn] r73580 - pypy/trunk/pypy/rlib Message-ID: <20100409113157.09749282BAD@codespeak.net> Author: afa Date: Fri Apr 9 13:31:56 2010 New Revision: 73580 Modified: pypy/trunk/pypy/rlib/libffi.py Log: ffi_abi is an enum, don't assume it's USHORT on all platforms. msvc indeed turns it into an int, and I've seen garbage values when running tests on top of pypy-c. Modified: pypy/trunk/pypy/rlib/libffi.py ============================================================================== --- pypy/trunk/pypy/rlib/libffi.py (original) +++ pypy/trunk/pypy/rlib/libffi.py Fri Apr 9 13:31:56 2010 @@ -128,6 +128,7 @@ FFI_TYPE_STRUCT = rffi_platform.ConstantInteger('FFI_TYPE_STRUCT') size_t = rffi_platform.SimpleType("size_t", rffi.ULONG) + ffi_abi = rffi_platform.SimpleType("ffi_abi", rffi.USHORT) ffi_type = rffi_platform.Struct('ffi_type', [('size', rffi.ULONG), ('alignment', rffi.USHORT), @@ -171,6 +172,7 @@ FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t +ffi_abi = cConfig.ffi_abi for name in type_names: locals()[name] = configure_simple_type(name) @@ -352,7 +354,7 @@ VOIDPP = rffi.CArrayPtr(rffi.VOIDP) -c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, rffi.USHORT, rffi.UINT, +c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, ffi_abi, rffi.UINT, FFI_TYPE_P, FFI_TYPE_PP], rffi.INT) if _WIN32: c_ffi_call_return_type = rffi.INT From afa at codespeak.net Fri Apr 9 13:36:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 13:36:04 +0200 (CEST) Subject: [pypy-svn] r73582 - in pypy/branch/cpython-extension/pypy: rlib translator/c/gcc Message-ID: <20100409113604.3628B282BAD@codespeak.net> Author: afa Date: Fri Apr 9 13:36:02 2010 New Revision: 73582 Modified: pypy/branch/cpython-extension/pypy/rlib/libffi.py pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py Log: merge r73577 and r73580 from trunk Modified: pypy/branch/cpython-extension/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/libffi.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/libffi.py Fri Apr 9 13:36:02 2010 @@ -128,6 +128,7 @@ FFI_TYPE_STRUCT = rffi_platform.ConstantInteger('FFI_TYPE_STRUCT') size_t = rffi_platform.SimpleType("size_t", rffi.ULONG) + ffi_abi = rffi_platform.SimpleType("ffi_abi", rffi.USHORT) ffi_type = rffi_platform.Struct('ffi_type', [('size', rffi.ULONG), ('alignment', rffi.USHORT), @@ -171,6 +172,7 @@ FFI_TYPE_P.TO.become(cConfig.ffi_type) size_t = cConfig.size_t +ffi_abi = cConfig.ffi_abi for name in type_names: locals()[name] = configure_simple_type(name) @@ -352,7 +354,7 @@ VOIDPP = rffi.CArrayPtr(rffi.VOIDP) -c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, rffi.USHORT, rffi.UINT, +c_ffi_prep_cif = external('ffi_prep_cif', [FFI_CIFP, ffi_abi, rffi.UINT, FFI_TYPE_P, FFI_TYPE_PP], rffi.INT) if _WIN32: c_ffi_call_return_type = rffi.INT Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/trackgcroot.py Fri Apr 9 13:36:02 2010 @@ -337,7 +337,7 @@ else: funcname = self.funcname while 1: - label = '__gcmap_%s__%s_%d' % (self.filetag, self.funcname, k) + label = '__gcmap_%s__%s_%d' % (self.filetag, funcname, k) if label not in self.labels: break k += 1 @@ -833,6 +833,8 @@ LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' OFFSET_LABELS = 0 + r_segmentstart = re.compile(r"[_A-Z]+\tSEGMENT$") + r_segmentend = re.compile(r"[_A-Z]+\tENDS$") r_functionstart = re.compile(r"; Function compile flags: ") r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") @@ -1012,7 +1014,8 @@ def process(self, iterlines, newfile, entrypoint='main', filename='?'): for in_function, lines in self.find_functions(iterlines): if in_function: - lines = self.process_function(lines, entrypoint, filename) + tracker = self.process_function(lines, entrypoint, filename) + lines = tracker.lines self.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') @@ -1040,25 +1043,24 @@ else: self.gcmaptable.extend(table) self.seen_main |= is_main - return tracker.lines + return tracker class ElfAssemblerParser(AssemblerParser): format = "elf" FunctionGcRootTracker = ElfFunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_function = False for line in iterlines: - if cls.FunctionGcRootTracker.r_functionstart.match(line): + if self.FunctionGcRootTracker.r_functionstart.match(line): assert not in_function, ( "missed the end of the previous function") yield False, functionlines in_function = True functionlines = [] functionlines.append(line) - if cls.FunctionGcRootTracker.r_functionend.match(line): + if self.FunctionGcRootTracker.r_functionend.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -1088,22 +1090,21 @@ ] r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_text = False in_function = False for n, line in enumerate(iterlines): - if cls.r_textstart.match(line): + if self.r_textstart.match(line): assert not in_text, "unexpected repeated .text start: %d" % n in_text = True - elif cls.r_sectionstart.match(line): + elif self.r_sectionstart.match(line): if in_function: yield in_function, functionlines functionlines = [] in_text = False in_function = False - elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -1121,17 +1122,16 @@ format = "mingw32" FunctionGcRootTracker = Mingw32FunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_text = False in_function = False for n, line in enumerate(iterlines): - if cls.r_textstart.match(line): + if self.r_textstart.match(line): in_text = True - elif cls.r_sectionstart.match(line): + elif self.r_sectionstart.match(line): in_text = False - elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -1143,19 +1143,41 @@ format = "msvc" FunctionGcRootTracker = MsvcFunctionGcRootTracker - @classmethod - def find_functions(cls, iterlines): + def find_functions(self, iterlines): functionlines = [] in_function = False + in_segment = False + ignore_public = False + self.inline_functions = {} for line in iterlines: - if cls.FunctionGcRootTracker.r_functionstart.match(line): + if line.startswith('; File '): + filename = line[:-1].split(' ', 2)[2] + ignore_public = ('wspiapi.h' in filename.lower()) + if ignore_public: + # this header define __inline functions, that are + # still marked as PUBLIC in the generated assembler + if line.startswith(';\tCOMDAT '): + funcname = line[:-1].split(' ', 1)[1] + self.inline_functions[funcname] = True + elif line.startswith('PUBLIC\t'): + funcname = line[:-1].split('\t')[1] + self.inline_functions[funcname] = True + + if self.FunctionGcRootTracker.r_segmentstart.match(line): + in_segment = True + elif self.FunctionGcRootTracker.r_functionstart.match(line): assert not in_function, ( "missed the end of the previous function") - yield False, functionlines in_function = True - functionlines = [] + if in_segment: + yield False, functionlines + functionlines = [] functionlines.append(line) - if cls.FunctionGcRootTracker.r_functionend.match(line): + if self.FunctionGcRootTracker.r_segmentend.match(line): + yield False, functionlines + in_segment = False + functionlines = [] + elif self.FunctionGcRootTracker.r_functionend.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -1181,14 +1203,18 @@ # compiler: every string or float constant is exported # with a name built after its value, and will conflict # with other modules. - if line.startswith("PUBLIC\t__real@"): - line = '; ' + line - if line.startswith("PUBLIC\t__mask@@"): - line = '; ' + line - elif line.startswith("PUBLIC\t??_C@"): - line = '; ' + line - elif line == "PUBLIC\t__$ArrayPad$\n": - line = '; ' + line + if line.startswith("PUBLIC\t"): + symbol = line[:-1].split()[1] + if symbol.startswith('__real@'): + line = '; ' + line + elif symbol.startswith("__mask@@"): + line = '; ' + line + elif symbol.startswith("??_C@"): + line = '; ' + line + elif symbol == "__$ArrayPad$": + line = '; ' + line + elif symbol in self.inline_functions: + line = '; ' + line # The msvc compiler writes "fucomip ST(1)" when the correct # syntax is "fucomip ST, ST(1)" @@ -1451,7 +1477,8 @@ parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) for in_function, lines in parser.find_functions(iterlines): if in_function: - lines = parser.process_function(lines, entrypoint, filename) + tracker = parser.process_function(lines, entrypoint, filename) + lines = tracker.lines parser.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') From trundle at codespeak.net Fri Apr 9 14:43:54 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Fri, 9 Apr 2010 14:43:54 +0200 (CEST) Subject: [pypy-svn] r73584 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100409124354.D0ABC282BDB@codespeak.net> Author: trundle Date: Fri Apr 9 14:43:51 2010 New Revision: 73584 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_floatobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Log: Add PyFloat_AS_DOUBLE and PyInt_AS_LONG. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/floatobject.py Fri Apr 9 14:43:51 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers +from pypy.module.cpyext.api import (CANNOT_FAIL, cpython_api, PyObject, + build_type_checkers) from pypy.interpreter.error import OperationError PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") @@ -12,6 +13,11 @@ def PyFloat_AsDouble(space, w_obj): return space.float_w(space.float(w_obj)) + at cpython_api([PyObject], lltype.Float, error=CANNOT_FAIL) +def PyFloat_AS_DOUBLE(space, w_float): + """Return a C double representation of the contents of w_float, but + without error checking.""" + return space.float_w(w_float) @cpython_api([PyObject], PyObject) def PyNumber_Float(space, w_obj): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Fri Apr 9 14:43:51 2010 @@ -21,6 +21,11 @@ there was an error, or whether the value just happened to be -1.""" return space.int_w(w_obj) + at cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) +def PyInt_AS_LONG(space, w_int): + """Return the value of the object w_int. No error checking is performed.""" + return space.int_w(w_int) + @cpython_api([PyObject], Py_ssize_t, error=-1) def PyInt_AsSsize_t(space, w_obj): """Will first attempt to cast the object to a PyIntObject or Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 9 14:43:51 2010 @@ -2051,12 +2051,6 @@ backward compatibility.""" raise NotImplementedError - at cpython_api([PyObject], {double}) -def PyFloat_AS_DOUBLE(space, pyfloat): - """Return a C double representation of the contents of pyfloat, but - without error checking.""" - raise NotImplementedError - @cpython_api([rffi.VOIDP_real], PyObject) def PyFloat_GetInfo(space, ): """Return a structseq instance which contains information about the @@ -3227,11 +3221,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], lltype.Signed) -def PyInt_AS_LONG(space, io): - """Return the value of the object io. No error checking is performed.""" - raise NotImplementedError - @cpython_api([PyObject], {unsigned long}) def PyInt_AsUnsignedLongMask(space, io): """Will first attempt to cast the object to a PyIntObject or Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_floatobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_floatobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_floatobject.py Fri Apr 9 14:43:51 2010 @@ -4,6 +4,7 @@ def test_floatobject(self, space, api): assert space.unwrap(api.PyFloat_FromDouble(3.14)) == 3.14 assert api.PyFloat_AsDouble(space.wrap(23.45)) == 23.45 + assert api.PyFloat_AS_DOUBLE(space.wrap(23.45)) == 23.45 assert api.PyFloat_AsDouble(space.w_None) == -1 api.PyErr_Clear() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Fri Apr 9 14:43:51 2010 @@ -8,7 +8,9 @@ assert not api.PyInt_Check(space.wrap((1, 2, 3))) for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]: x = api.PyInt_AsLong(space.wrap(i)) + y = api.PyInt_AS_LONG(space.wrap(i)) assert x == i + assert y == i w_x = api.PyInt_FromLong(x + 1) assert space.type(w_x) is space.w_int assert space.eq_w(w_x, space.wrap(i + 1)) From trundle at codespeak.net Fri Apr 9 14:54:19 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Fri, 9 Apr 2010 14:54:19 +0200 (CEST) Subject: [pypy-svn] r73585 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100409125419.6E0CD282BAD@codespeak.net> Author: trundle Date: Fri Apr 9 14:54:16 2010 New Revision: 73585 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Log: Fix PyInt_AsLong (coercion). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Fri Apr 9 14:54:16 2010 @@ -19,7 +19,7 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" - return space.int_w(w_obj) + return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) def PyInt_AS_LONG(space, w_int): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Fri Apr 9 14:54:16 2010 @@ -18,3 +18,9 @@ assert api.PyInt_AsLong(space.w_None) == -1 assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + + def test_coerce(self, space, api): + class Coerce(object): + def __int__(self): + return 42 + assert api.PyInt_AsLong(space.wrap(Coerce())) == 42 From afa at codespeak.net Fri Apr 9 18:31:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 18:31:43 +0200 (CEST) Subject: [pypy-svn] r73589 - in pypy/trunk/pypy/lib: _ctypes app_test/ctypes_tests Message-ID: <20100409163143.ABF89282BAD@codespeak.net> Author: afa Date: Fri Apr 9 18:31:41 2010 New Revision: 73589 Modified: pypy/trunk/pypy/lib/_ctypes/pointer.py pypy/trunk/pypy/lib/app_test/ctypes_tests/test_keepalive.py Log: ctypes.cast between pointers must keep alive the converted objects Modified: pypy/trunk/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/trunk/pypy/lib/_ctypes/pointer.py (original) +++ pypy/trunk/pypy/lib/_ctypes/pointer.py Fri Apr 9 18:31:41 2010 @@ -126,5 +126,13 @@ raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) result = tp() + + # The casted objects '_objects' member: + # It must certainly contain the source objects one. + # It must contain the source object itself. + if obj._ensure_objects() is not None: + result._objects = {keepalive_key(0): obj._objects, + keepalive_key(1): obj} + result._buffer[0] = obj._buffer[0] return result Modified: pypy/trunk/pypy/lib/app_test/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/ctypes_tests/test_keepalive.py (original) +++ pypy/trunk/pypy/lib/app_test/ctypes_tests/test_keepalive.py Fri Apr 9 18:31:41 2010 @@ -117,7 +117,14 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) assert cf._objects == {'0':cf} - + + def test_cfunc_cast(self): + def f(): + pass + cf = CFUNCTYPE(c_int, c_int)(f) + p1 = cast(cf, c_void_p) + assert p1._objects == {'1': cf, '0': {'0': cf}} + def test_array_of_struct_with_pointer(self): class S(Structure): _fields_ = [('x', c_int)] From jandem at codespeak.net Fri Apr 9 18:59:51 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 9 Apr 2010 18:59:51 +0200 (CEST) Subject: [pypy-svn] r73590 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100409165951.08A75282BAD@codespeak.net> Author: jandem Date: Fri Apr 9 18:59:48 2010 New Revision: 73590 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: Support METH_O functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 9 18:59:48 2010 @@ -56,7 +56,8 @@ constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING -METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS METH_VARARGS METH_KEYWORDS +METH_COEXIST METH_STATIC METH_CLASS +METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS """.split() for name in constant_names: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Fri Apr 9 18:59:48 2010 @@ -9,7 +9,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ - cpython_struct, METH_KEYWORDS + cpython_struct, METH_KEYWORDS, METH_O from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated @@ -41,10 +41,18 @@ if space.is_true(w_kw) and not flags & METH_KEYWORDS: raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) - # XXX support METH_NOARGS, METH_O + # XXX support METH_NOARGS if flags & METH_KEYWORDS: func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) return generic_cpy_call(space, func, w_self, w_args, w_kw) + elif flags & METH_O: + if len(w_args.wrappeditems) != 1: + raise OperationError(space.w_TypeError, + space.wrap("%s() takes exactly one argument (%d given)" % ( + rffi.charp2str(self.ml.c_ml_name), + len(w_args.wrappeditems)))) + w_arg = w_args.wrappeditems[0] + return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) else: return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_args) Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Fri Apr 9 18:59:48 2010 @@ -0,0 +1,16 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestGetargs(AppTestCpythonExtensionBase): + def test_METH_O(self): + mod = self.import_extension('foo', [ + ('getarg', 'METH_O', + ''' + Py_INCREF(args); + return args; + ''' + ), + ]) + assert mod.getarg(1) == 1 + raises(TypeError, mod.getarg) + raises(TypeError, mod.getarg, 1, 1) From jandem at codespeak.net Fri Apr 9 19:04:54 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 9 Apr 2010 19:04:54 +0200 (CEST) Subject: [pypy-svn] r73591 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100409170454.A9E7D282BAD@codespeak.net> Author: jandem Date: Fri Apr 9 19:04:51 2010 New Revision: 73591 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Set extension_name to full_name and fix a typo in test Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Fri Apr 9 19:04:51 2010 @@ -1,7 +1,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -class AppTestGetargs(AppTestCpythonExtensionBase): +class AppTestMethodObject(AppTestCpythonExtensionBase): def test_METH_O(self): mod = self.import_extension('foo', [ ('getarg', 'METH_O', Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 9 19:04:51 2010 @@ -212,7 +212,7 @@ module_name, extension_name = rsplit(full_name, ".", 1) dict_w["__module__"] = space.wrap(module_name) else: - extension_name = None + extension_name = full_name W_TypeObject.__init__(self, space, extension_name, bases_w or [space.w_object], dict_w) From jandem at codespeak.net Fri Apr 9 19:23:15 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 9 Apr 2010 19:23:15 +0200 (CEST) Subject: [pypy-svn] r73593 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100409172315.7115C282BAD@codespeak.net> Author: jandem Date: Fri Apr 9 19:23:12 2010 New Revision: 73593 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: add Py_CLEAR macro Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Fri Apr 9 19:23:12 2010 @@ -42,6 +42,15 @@ #define Py_XINCREF(ob) (Py_IncRef(ob)) #define Py_XDECREF(ob) (Py_DecRef(ob)) +#define Py_CLEAR(op) \ + do { \ + if (op) { \ + PyObject *_py_tmp = (PyObject *)(op); \ + (op) = NULL; \ + Py_DECREF(_py_tmp); \ + } \ + } while (0) + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) From trundle at codespeak.net Fri Apr 9 19:30:24 2010 From: trundle at codespeak.net (trundle at codespeak.net) Date: Fri, 9 Apr 2010 19:30:24 +0200 (CEST) Subject: [pypy-svn] r73594 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100409173024.1CE43282BAD@codespeak.net> Author: trundle Date: Fri Apr 9 19:30:22 2010 New Revision: 73594 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: Implement wrap_lenfunc. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 9 19:30:22 2010 @@ -730,6 +730,8 @@ state.check_and_raise_exception() return ret + else: + return space.wrap(result) finally: if decref_args: for ref in to_decref: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Fri Apr 9 19:30:22 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import generic_cpy_call, cpython_api, \ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ - ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc + ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt @@ -49,6 +49,11 @@ func_target = rffi.cast(ternaryfunc, func) return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) +def wrap_lenfunc(space, w_self, w_args, func): + func_len = rffi.cast(lenfunc, func) + check_num_args(space, w_args, 0) + return generic_cpy_call(space, func_len, w_self) + @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check From afa at codespeak.net Fri Apr 9 19:30:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 9 Apr 2010 19:30:55 +0200 (CEST) Subject: [pypy-svn] r73595 - in pypy/branch/cpython-extension/pypy/lib: _ctypes app_test/ctypes_tests Message-ID: <20100409173055.C2CD6282BAD@codespeak.net> Author: afa Date: Fri Apr 9 19:30:53 2010 New Revision: 73595 Modified: pypy/branch/cpython-extension/pypy/lib/_ctypes/pointer.py pypy/branch/cpython-extension/pypy/lib/app_test/ctypes_tests/test_keepalive.py Log: Merge r73589 from trunk Modified: pypy/branch/cpython-extension/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/branch/cpython-extension/pypy/lib/_ctypes/pointer.py (original) +++ pypy/branch/cpython-extension/pypy/lib/_ctypes/pointer.py Fri Apr 9 19:30:53 2010 @@ -126,5 +126,13 @@ raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) result = tp() + + # The casted objects '_objects' member: + # It must certainly contain the source objects one. + # It must contain the source object itself. + if obj._ensure_objects() is not None: + result._objects = {keepalive_key(0): obj._objects, + keepalive_key(1): obj} + result._buffer[0] = obj._buffer[0] return result Modified: pypy/branch/cpython-extension/pypy/lib/app_test/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/branch/cpython-extension/pypy/lib/app_test/ctypes_tests/test_keepalive.py (original) +++ pypy/branch/cpython-extension/pypy/lib/app_test/ctypes_tests/test_keepalive.py Fri Apr 9 19:30:53 2010 @@ -117,7 +117,14 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) assert cf._objects == {'0':cf} - + + def test_cfunc_cast(self): + def f(): + pass + cf = CFUNCTYPE(c_int, c_int)(f) + p1 = cast(cf, c_void_p) + assert p1._objects == {'1': cf, '0': {'0': cf}} + def test_array_of_struct_with_pointer(self): class S(Structure): _fields_ = [('x', c_int)] From jandem at codespeak.net Fri Apr 9 19:44:54 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 9 Apr 2010 19:44:54 +0200 (CEST) Subject: [pypy-svn] r73597 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100409174454.8DDC4282BAD@codespeak.net> Author: jandem Date: Fri Apr 9 19:44:53 2010 New Revision: 73597 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: RPythonify Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Fri Apr 9 19:44:53 2010 @@ -13,7 +13,7 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated - +from pypy.objspace.std.tupleobject import W_TupleObject PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject], PyObject)) @@ -46,6 +46,7 @@ func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) return generic_cpy_call(space, func, w_self, w_args, w_kw) elif flags & METH_O: + assert isinstance(w_args, W_TupleObject) if len(w_args.wrappeditems) != 1: raise OperationError(space.w_TypeError, space.wrap("%s() takes exactly one argument (%d given)" % ( From jandem at codespeak.net Fri Apr 9 20:22:33 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 9 Apr 2010 20:22:33 +0200 (CEST) Subject: [pypy-svn] r73600 - in pypy/branch/cpython-extension/pypy/module/cpyext: . src Message-ID: <20100409182233.90F32282BAD@codespeak.net> Author: jandem Date: Fri Apr 9 20:22:30 2010 New Revision: 73600 Added: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Add stubsactive.py as suggested by xorAxAx Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Fri Apr 9 20:22:30 2010 @@ -61,6 +61,7 @@ import pypy.module.cpyext.sysmodule import pypy.module.cpyext.number import pypy.module.cpyext.sliceobject +import pypy.module.cpyext.stubsactive # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c Fri Apr 9 20:22:30 2010 @@ -840,7 +840,7 @@ (hashfunc)buffer_hash, /* tp_hash */ 0, /* tp_call */ (reprfunc)buffer_str, /* tp_str */ - 0, //XXX PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &buffer_as_buffer, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GETCHARBUFFER, /* tp_flags */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 9 20:22:30 2010 @@ -4308,16 +4308,6 @@ expression o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyObject_GenericGetAttr(space, o, name): - """Generic attribute getter function that is meant to be put into a type - object's tp_getattro slot. It looks for a descriptor in the dictionary - of classes in the object's MRO as well as an attribute in the object's - __dict__ (if present). As outlined in descriptors, data - descriptors take preference over instance attributes, while non-data - descriptors don't. Otherwise, an AttributeError is raised.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real) def PyObject_SetAttrString(space, o, attr_name, v): """Set the value of the attribute named attr_name, for object o, to the value Added: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Fri Apr 9 20:22:30 2010 @@ -0,0 +1,13 @@ +from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL +from pypy.rpython.lltypesystem import rffi, lltype + + at cpython_api([PyObject, PyObject], PyObject) +def PyObject_GenericGetAttr(space, o, name): + """Generic attribute getter function that is meant to be put into a type + object's tp_getattro slot. It looks for a descriptor in the dictionary + of classes in the object's MRO as well as an attribute in the object's + __dict__ (if present). As outlined in descriptors, data + descriptors take preference over instance attributes, while non-data + descriptors don't. Otherwise, an AttributeError is raised.""" + raise NotImplementedError From fijal at codespeak.net Fri Apr 9 22:03:36 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Apr 2010 22:03:36 +0200 (CEST) Subject: [pypy-svn] r73602 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100409200336.3A545282BAD@codespeak.net> Author: fijal Date: Fri Apr 9 22:03:33 2010 New Revision: 73602 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: don't use debug wrapper when we're translated Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 9 22:03:33 2010 @@ -342,7 +342,7 @@ # inserted exactly here by the varargs specializer llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py boxed_args = () - if DEBUG_WRAPPER: + if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] @@ -357,7 +357,7 @@ state = space.fromcache(State) try: retval = callable(space, *boxed_args) - if DEBUG_WRAPPER: + if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, " DONE" except OperationError, e: failed = True From fijal at codespeak.net Fri Apr 9 22:09:12 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Apr 2010 22:09:12 +0200 (CEST) Subject: [pypy-svn] r73603 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100409200912.47152282BAD@codespeak.net> Author: fijal Date: Fri Apr 9 22:09:10 2010 New Revision: 73603 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: fix that, untested! Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Fri Apr 9 22:09:10 2010 @@ -139,7 +139,7 @@ the Python codec registry. Return NULL if an exception was raised by the codec.""" if not PyUnicode_Check(space, w_unicode): - PyErr_BadArgument() + PyErr_BadArgument(space) w_encoding = w_errors = None if encoding: From fijal at codespeak.net Fri Apr 9 22:29:52 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Apr 2010 22:29:52 +0200 (CEST) Subject: [pypy-svn] r73604 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100409202952.849EC282BAD@codespeak.net> Author: fijal Date: Fri Apr 9 22:29:50 2010 New Revision: 73604 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: refactor out error handling Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py Fri Apr 9 22:29:50 2010 @@ -5,6 +5,8 @@ from pypy.module.cpyext import api from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, check_and_print_leaks PyObject = api.PyObject +from pypy.interpreter.error import OperationError +from pypy.module.cpyext.state import State @api.cpython_api([PyObject], lltype.Void) def PyPy_GetWrapped(space, w_arg): @@ -23,6 +25,18 @@ cls.api = CAPI() CAPI.__dict__.update(api.INTERPLEVEL_API) + def raises(self, space, api, expected_exc, f, *args): + if not callable(f): + raise Exception("%s is not callable" % (f,)) + f(*args) + state = space.fromcache(State) + tp = state.exc_type + if not tp: + raise Exception("DID NOT RAISE") + if getattr(space, 'w_' + expected_exc.__name__) is not tp: + raise Exception("Wrong exception") + state.clear_exception() + def setup_method(self, func): freeze_refcnts(self) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Fri Apr 9 22:29:50 2010 @@ -36,14 +36,16 @@ assert array[i] == char assert array2[i] == char assert array3[i] == char - raises(TypeError, api.PyUnicode_AsUnicode(space.wrap('spam'))) - api.PyErr_Clear() + self.raises(space, api, TypeError, api.PyUnicode_AsUnicode, + space.wrap('spam')) utf_8 = rffi.str2charp('utf-8') encoded = api.PyUnicode_AsEncodedString(space.wrap(u'sp?m'), utf_8, None) assert space.unwrap(encoded) == 'sp\xc3\xa4m' - raises(TypeError, api.PyUnicode_AsEncodedString, + self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString, + space.newtuple([1, 2, 3]), None, None) + self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString, space.wrap(''), None, None) rffi.free_charp(utf_8) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Fri Apr 9 22:29:50 2010 @@ -102,7 +102,7 @@ ref_unicode.c_buffer = rffi.unicode2wcharp(u) return ref_unicode.c_buffer - at cpython_api([PyObject], rffi.CWCHARP, error=0) + at cpython_api([PyObject], rffi.CWCHARP, error=lltype.nullptr(rffi.CWCHARP.TO)) def PyUnicode_AsUnicode(space, ref): """Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer, NULL if unicode is not a Unicode object.""" From fijal at codespeak.net Fri Apr 9 22:50:31 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 9 Apr 2010 22:50:31 +0200 (CEST) Subject: [pypy-svn] r73605 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100409205031.050F4282BAD@codespeak.net> Author: fijal Date: Fri Apr 9 22:50:29 2010 New Revision: 73605 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: I think this is better test (fails) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Fri Apr 9 22:50:29 2010 @@ -9,7 +9,7 @@ class TestLongObject(BaseApiTest): def test_FromLong(self, space, api): value = api.PyLong_FromLong(3) - assert isinstance(value, W_IntObject) + assert isinstance(value, W_LongObject) assert space.unwrap(value) == 3 value = api.PyLong_FromLong(sys.maxint + 1) From afa at codespeak.net Sat Apr 10 00:18:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 10 Apr 2010 00:18:51 +0200 (CEST) Subject: [pypy-svn] r73606 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100409221851.516FD282BAD@codespeak.net> Author: afa Date: Sat Apr 10 00:18:48 2010 New Revision: 73606 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Fix translation. This member is only used in tests. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Sat Apr 10 00:18:48 2010 @@ -34,7 +34,8 @@ for func in api.INIT_FUNCTIONS: func() state.check_and_raise_exception() - state.non_heaptypes[:] = [] + if not we_are_translated(): + state.non_heaptypes[:] = [] # import these modules to register api functions by side-effect import pypy.module.cpyext.pyobject From xoraxax at codespeak.net Sat Apr 10 12:21:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 10 Apr 2010 12:21:29 +0200 (CEST) Subject: [pypy-svn] r73607 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410102129.60D6F282BAD@codespeak.net> Author: xoraxax Date: Sat Apr 10 12:21:27 2010 New Revision: 73607 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add a refactoring comment. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 12:21:27 2010 @@ -225,7 +225,7 @@ INTERPLEVEL_API = {} FUNCTIONS = {} FUNCTIONS_STATIC = {} -FUNCTIONS_C = [ +FUNCTIONS_C = [ # XXX rename to SYMBOLS_C 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', From xoraxax at codespeak.net Sat Apr 10 12:21:46 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 10 Apr 2010 12:21:46 +0200 (CEST) Subject: [pypy-svn] r73608 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410102146.C5CDB282BAD@codespeak.net> Author: xoraxax Date: Sat Apr 10 12:21:45 2010 New Revision: 73608 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/state.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Only allocate new_method_def once. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/state.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/state.py Sat Apr 10 12:21:45 2010 @@ -11,6 +11,7 @@ self.reset() def reset(self): + from pypy.module.cpyext.modsupport import PyMethodDef self.py_objects_w2r = {} # { w_obj -> raw PyObject } self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } self.borrow_mapping = {} # { addr of container -> { addr of containee -> None } } @@ -19,6 +20,7 @@ self.last_container = 0 # addr of last container self.exc_type = None self.exc_value = None + self.new_method_def = lltype.nullptr(PyMethodDef) def _freeze_(self): assert not self.borrowed_objects and not self.borrow_mapping Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sat Apr 10 12:21:45 2010 @@ -165,11 +165,15 @@ @specialize.memo() def get_new_method_def(space): + state = space.fromcache(State) + if state.new_method_def: + return state.new_method_def from pypy.module.cpyext.modsupport import PyMethodDef ptr = lltype.malloc(PyMethodDef, flavor="raw", zero=True) ptr.c_ml_name = rffi.str2charp("__new__") rffi.setintfield(ptr, 'c_ml_flags', METH_VARARGS | METH_KEYWORDS) ptr.c_ml_doc = rffi.str2charp("T.__new__(S, ...) -> a new object with type S, a subtype of T") + state.new_method_def = ptr return ptr def setup_new_method_def(space): From jandem at codespeak.net Sat Apr 10 13:17:02 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 13:17:02 +0200 (CEST) Subject: [pypy-svn] r73609 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410111702.AEEF9282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 13:17:01 2010 New Revision: 73609 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Add PyObject_Repr, with tests Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sat Apr 10 13:17:01 2010 @@ -105,3 +105,10 @@ op.c_ob_refcnt = 1 return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise + at cpython_api([PyObject], PyObject) +def PyObject_Repr(space, w_obj): + """Compute a string representation of object o. Returns the string + representation on success, NULL on failure. This is the equivalent of the + Python expression repr(o). Called by the repr() built-in function and + by reverse quotes.""" + return space.repr(w_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 13:17:01 2010 @@ -64,3 +64,7 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 + + def test_repr(self, space, api): + w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" From jandem at codespeak.net Sat Apr 10 13:48:55 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 13:48:55 +0200 (CEST) Subject: [pypy-svn] r73610 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100410114855.40C03282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 13:48:53 2010 New Revision: 73610 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: add PyObject_RichCompare{Bool}, with tests Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 13:48:53 2010 @@ -59,6 +59,7 @@ METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS +Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE """.split() for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sat Apr 10 13:48:53 2010 @@ -57,6 +57,21 @@ #define Py_None (&_Py_NoneStruct) +/* +Py_NotImplemented is a singleton used to signal that an operation is +not implemented for a given type combination. +*/ +PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ +#define Py_NotImplemented (&_Py_NotImplementedStruct) + +/* Rich comparison opcodes */ +#define Py_LT 0 +#define Py_LE 1 +#define Py_EQ 2 +#define Py_NE 3 +#define Py_GT 4 +#define Py_GE 5 + struct _typeobject; typedef void (*freefunc)(void *); typedef void (*destructor)(PyObject *); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sat Apr 10 13:48:53 2010 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE + Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ + Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State @@ -112,3 +113,31 @@ Python expression repr(o). Called by the repr() built-in function and by reverse quotes.""" return space.repr(w_obj) + + at cpython_api([PyObject, PyObject, rffi.INT_real], PyObject) +def PyObject_RichCompare(space, w_o1, w_o2, opid): + """Compare the values of o1 and o2 using the operation specified by opid, + which must be one of Py_LT, Py_LE, Py_EQ, + Py_NE, Py_GT, or Py_GE, corresponding to <, + <=, ==, !=, >, or >= respectively. This is the equivalent of + the Python expression o1 op o2, where op is the operator corresponding + to opid. Returns the value of the comparison on success, or NULL on failure.""" + if opid == Py_LT: return space.lt(w_o1, w_o2) + if opid == Py_LE: return space.le(w_o1, w_o2) + if opid == Py_EQ: return space.eq(w_o1, w_o2) + if opid == Py_NE: return space.ne(w_o1, w_o2) + if opid == Py_GT: return space.gt(w_o1, w_o2) + if opid == Py_GE: return space.ge(w_o1, w_o2) + PyErr_BadInternalCall(space) + + at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) +def PyObject_RichCompareBool(space, ref1, ref2, opid): + """Compare the values of o1 and o2 using the operation specified by opid, + which must be one of Py_LT, Py_LE, Py_EQ, + Py_NE, Py_GT, or Py_GE, corresponding to <, + <=, ==, !=, >, or >= respectively. Returns -1 on error, + 0 if the result is false, 1 otherwise. This is the equivalent of the + Python expression o1 op o2, where op is the operator corresponding to + opid.""" + w_res = PyObject_RichCompare(space, ref1, ref2, opid) + return PyObject_IsTrue(space, w_res) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 13:48:53 2010 @@ -2,7 +2,8 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.rpython.lltypesystem import rffi, lltype - +from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ + Py_GE, Py_GT class TestObject(BaseApiTest): def test_IsTrue(self, space, api): @@ -53,7 +54,6 @@ lltype.free(charp1, flavor="raw") lltype.free(charp2, flavor="raw") - def test_getitem(self, space, api): w_t = space.wrap((1, 2, 3, 4, 5)) assert space.unwrap(api.PyObject_GetItem(w_t, space.wrap(3))) == 4 @@ -68,3 +68,28 @@ def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" + + def test_RichCompare(self, space, api): + w_i1 = space.wrap(1) + w_i2 = space.wrap(2) + w_i3 = space.wrap(3) + + def compare(w_o1, w_o2, opid): + res = api.PyObject_RichCompareBool(w_o1, w_o2, opid) + w_res = api.PyObject_RichCompare(w_o1, w_o2, opid) + assert space.is_true(w_res) == res + return res + + def test_compare(o1, o2): + w_o1 = space.wrap(o1) + w_o2 = space.wrap(o2) + + for opid, expected in [ + (Py_LT, o1 < o2), (Py_LE, o1 <= o2), + (Py_NE, o1 != o2), (Py_EQ, o1 == o2), + (Py_GT, o1 > o2), (Py_GE, o1 >= o2)]: + assert compare(w_o1, w_o2, opid) == expected + + test_compare(1, 2) + test_compare(2, 2) + test_compare('2', '1') From jandem at codespeak.net Sat Apr 10 13:51:10 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 13:51:10 +0200 (CEST) Subject: [pypy-svn] r73611 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100410115110.16DBF282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 13:51:09 2010 New Revision: 73611 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: remove unused vars from test Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 13:51:09 2010 @@ -70,10 +70,6 @@ assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" def test_RichCompare(self, space, api): - w_i1 = space.wrap(1) - w_i2 = space.wrap(2) - w_i3 = space.wrap(3) - def compare(w_o1, w_o2, opid): res = api.PyObject_RichCompareBool(w_o1, w_o2, opid) w_res = api.PyObject_RichCompare(w_o1, w_o2, opid) From jandem at codespeak.net Sat Apr 10 14:01:47 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 14:01:47 +0200 (CEST) Subject: [pypy-svn] r73612 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410120147.7BA64282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 14:01:46 2010 New Revision: 73612 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Test PyObject_RichCompare with invalid opid Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sat Apr 10 14:01:46 2010 @@ -6,7 +6,7 @@ from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject -from pypy.module.cpyext.pyerrors import PyErr_NoMemory +from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject import pypy.module.__builtin__.operation as operation @@ -140,4 +140,4 @@ Python expression o1 op o2, where op is the operator corresponding to opid.""" w_res = PyObject_RichCompare(space, ref1, ref2, opid) - return PyObject_IsTrue(space, w_res) + return int(space.is_true(w_res)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 14:01:46 2010 @@ -89,3 +89,8 @@ test_compare(1, 2) test_compare(2, 2) test_compare('2', '1') + + w_i = space.wrap(1) + assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 + assert api.PyErr_Occurred() is space.w_SystemError + api.PyErr_Clear() From jandem at codespeak.net Sat Apr 10 14:50:24 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 14:50:24 +0200 (CEST) Subject: [pypy-svn] r73614 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410125024.EFF05282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 14:50:23 2010 New Revision: 73614 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Add PyString_Concat{AndDel} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Sat Apr 10 14:50:23 2010 @@ -88,3 +88,31 @@ Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 + + at cpython_api([PyObjectP, PyObject], lltype.Void) +def PyString_Concat(space, ref, w_newpart): + """Create a new string object in *string containing the contents of newpart + appended to string; the caller will own the new reference. The reference to + the old value of string will be stolen. If the new string cannot be created, + the old reference to string will still be discarded and the value of + *string will be set to NULL; the appropriate exception will be set.""" + + if not ref[0]: + return + + if w_newpart is None or not PyString_Check(space, ref[0]) or \ + not PyString_Check(space, w_newpart): + Py_DecRef(space, ref[0]) + ref[0] = lltype.nullptr(PyObject.TO) + return + w_str = from_ref(space, ref[0]) + w_newstr = space.add(w_str, w_newpart) + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, w_newstr) + + at cpython_api([PyObjectP, PyObject], lltype.Void) +def PyString_ConcatAndDel(space, ref, newpart): + """Create a new string object in *string containing the contents of newpart + appended to string. This version decrements the reference count of newpart.""" + PyString_Concat(space, ref, newpart) + Py_DecRef(space, newpart) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Sat Apr 10 14:50:23 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.stringobject import new_empty_str from pypy.module.cpyext.api import PyStringObject, PyObjectP, PyObject -from pypy.module.cpyext.pyobject import Py_DecRef +from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref import py import sys @@ -177,3 +177,30 @@ assert c_buf.c_bf_getreadbuffer(py_obj, 0, ref) == 10 lltype.free(ref, flavor='raw') Py_DecRef(space, py_obj) + + def test_Concat(self, space, api): + ref = make_ref(space, space.wrap('abc')) + ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + ptr[0] = ref + api.PyString_Concat(ptr, space.wrap('def')) + assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' + api.PyString_Concat(ptr, space.w_None) + assert not ptr[0] + ptr[0] = lltype.nullptr(PyObject.TO) + api.PyString_Concat(ptr, space.wrap('def')) # should not crash + lltype.free(ptr, flavor='raw') + + def test_ConcatAndDel(self, space, api): + ref1 = make_ref(space, space.wrap('abc')) + ref2 = make_ref(space, space.wrap('def')) + ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') + ptr[0] = ref1 + api.PyString_ConcatAndDel(ptr, ref2) + assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' + assert ref2.c_ob_refcnt == 0 + Py_DecRef(space, ptr[0]) + ptr[0] = lltype.nullptr(PyObject.TO) + ref2 = make_ref(space, space.wrap('foo')) + api.PyString_ConcatAndDel(ptr, ref2) # should not crash + assert ref2.c_ob_refcnt == 0 + lltype.free(ptr, flavor='raw') From jandem at codespeak.net Sat Apr 10 14:53:13 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 14:53:13 +0200 (CEST) Subject: [pypy-svn] r73616 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410125313.26738282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 14:53:11 2010 New Revision: 73616 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add _PyArg_NoKeywords to FUNCTIONS_C to prevent segfaults Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 14:53:11 2010 @@ -232,6 +232,7 @@ 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', + '_PyArg_NoKeywords', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur From jandem at codespeak.net Sat Apr 10 15:01:54 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 15:01:54 +0200 (CEST) Subject: [pypy-svn] r73617 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410130154.72065282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 15:01:53 2010 New Revision: 73617 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: Add PyList_Size, with tests Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Sat Apr 10 15:01:53 2010 @@ -70,3 +70,16 @@ code for properly supporting 64-bit systems.""" assert isinstance(w_list, W_ListObject) return len(w_list.wrappeditems) + + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyList_Size(space, ref): + """Return the length of the list object in list; this is equivalent to + len(list) on a list object. + + This function returned an int. This might require changes in + your code for properly supporting 64-bit systems.""" + if not PyList_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected list object")) + return PyList_GET_SIZE(space, ref) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Sat Apr 10 15:01:53 2010 @@ -25,6 +25,13 @@ assert api.PyList_GET_SIZE(l) == 0 api.PyList_Append(l, space.wrap(3)) assert api.PyList_GET_SIZE(l) == 1 + + def test_size(self, space, api): + l = space.newlist([space.w_None, space.w_None]) + assert api.PyList_Size(l) == 2 + assert api.PyList_Size(space.w_None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() class AppTestListObject(AppTestCpythonExtensionBase): def test_listobject(self): From jandem at codespeak.net Sat Apr 10 15:16:42 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 15:16:42 +0200 (CEST) Subject: [pypy-svn] r73618 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100410131642.693F2282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 15:16:40 2010 New Revision: 73618 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/warnings.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Add PyErr_WarnPy3k macro Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sat Apr 10 15:16:40 2010 @@ -75,6 +75,7 @@ #include "object.h" #include "pyport.h" +#include "warnings.h" #include #include Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/warnings.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/warnings.h Sat Apr 10 15:16:40 2010 @@ -0,0 +1,12 @@ +#ifndef Py_WARNINGS_H +#define Py_WARNINGS_H +#ifdef __cplusplus +extern "C" { +#endif + +#define PyErr_WarnPy3k(msg, stacklevel) 0 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_WARNINGS_H */ From jandem at codespeak.net Sat Apr 10 15:43:10 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 15:43:10 +0200 (CEST) Subject: [pypy-svn] r73620 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410134310.B6394282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 15:43:09 2010 New Revision: 73620 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Add PyObject_TypeCheck, with tests Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sat Apr 10 15:43:09 2010 @@ -141,3 +141,13 @@ opid.""" w_res = PyObject_RichCompare(space, ref1, ref2, opid) return int(space.is_true(w_res)) + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyObject_TypeCheck(space, w_obj, w_type): + """Return true if the object o is of type type or a subtype of type. Both + parameters must be non-NULL. + """ + w_obj_type = space.type(w_obj) + assert isinstance(w_type, W_TypeObject) + return int(space.is_w(w_obj_type, w_type) or + space.is_true(space.issubtype(w_obj_type, w_type))) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 15:43:09 2010 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ Py_GE, Py_GT +from pypy.module.cpyext.pyobject import make_ref class TestObject(BaseApiTest): def test_IsTrue(self, space, api): @@ -94,3 +95,8 @@ assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1 assert api.PyErr_Occurred() is space.w_SystemError api.PyErr_Clear() + + def test_TypeCheck(self, space, api): + assert api.PyObject_TypeCheck(space.wrap(1), space.w_int) + assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_str) + assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_object) From jandem at codespeak.net Sat Apr 10 16:04:55 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:04:55 +0200 (CEST) Subject: [pypy-svn] r73622 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410140455.6CCC9282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:04:53 2010 New Revision: 73622 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add PyNotImplemented_Type to GLOBALS as well Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 16:04:53 2010 @@ -253,6 +253,7 @@ "Unicode": "space.w_unicode", 'Bool': 'space.w_bool', 'None': 'space.type(space.w_None)', + 'NotImplemented': 'space.w_NotImplemented', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From jandem at codespeak.net Sat Apr 10 16:08:20 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:08:20 +0200 (CEST) Subject: [pypy-svn] r73623 - in pypy/branch/cpython-extension/pypy/module/cpyext: include test Message-ID: <20100410140820.05027282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:08:19 2010 New Revision: 73623 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Add some macros from CPython, remove unused import Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sat Apr 10 16:08:19 2010 @@ -447,6 +447,23 @@ (typeobj), (n)) ) */ +#define PyObject_GC_New(type, typeobj) \ + ( (type *) _PyObject_GC_New(typeobj) ) + +/* Utility macro to help write tp_traverse functions. + * To use this macro, the tp_traverse function must name its arguments + * "visit" and "arg". This is intended to keep tp_traverse functions + * looking as much alike as possible. + */ +#define Py_VISIT(op) \ + do { \ + if (op) { \ + int vret = visit((PyObject *)(op), arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Sat Apr 10 16:08:19 2010 @@ -7,8 +7,44 @@ /* XXX use obmalloc like cpython and pypy do, otherwise we might get segfaults */ #define PyObject_MALLOC PyMem_MALLOC -//#define PyObject_REALLOC PyMem_REALLOC +#define PyObject_REALLOC PyMem_REALLOC #define PyObject_FREE PyMem_FREE #define PyMem_Malloc PyMem_MALLOC #define PyMem_Free PyMem_FREE + +/* + * Type-oriented memory interface + * ============================== + * + * Allocate memory for n objects of the given type. Returns a new pointer + * or NULL if the request was too large or memory allocation failed. Use + * these macros rather than doing the multiplication yourself so that proper + * overflow checking is always done. + */ + +#define PyMem_New(type, n) \ + ( ((n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ + ( (type *) PyMem_Malloc((n) * sizeof(type)) ) ) +#define PyMem_NEW(type, n) \ + ( ((n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ + ( (type *) PyMem_MALLOC((n) * sizeof(type)) ) ) + +/* + * The value of (p) is always clobbered by this macro regardless of success. + * The caller MUST check if (p) is NULL afterwards and deal with the memory + * error if so. This means the original value of (p) MUST be saved for the + * caller's memory error handler to not lose track of it. + */ +#define PyMem_Resize(p, type, n) \ + ( (p) = ((n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ + (type *) PyMem_Realloc((p), (n) * sizeof(type)) ) +#define PyMem_RESIZE(p, type, n) \ + ( (p) = ((n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ + (type *) PyMem_REALLOC((p), (n) * sizeof(type)) ) + +/* PyMem{Del,DEL} are left over from ancient days, and shouldn't be used + * anymore. They're just confusing aliases for PyMem_{Free,FREE} now. + */ +#define PyMem_Del PyMem_Free +#define PyMem_DEL PyMem_FREE Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sat Apr 10 16:08:19 2010 @@ -4,7 +4,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ Py_GE, Py_GT -from pypy.module.cpyext.pyobject import make_ref class TestObject(BaseApiTest): def test_IsTrue(self, space, api): From jandem at codespeak.net Sat Apr 10 16:16:58 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:16:58 +0200 (CEST) Subject: [pypy-svn] r73624 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410141658.2A66D282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:16:56 2010 New Revision: 73624 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Fix NotImplemented type the right way Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 16:16:56 2010 @@ -239,6 +239,7 @@ '_Py_NoneStruct#': ('PyObject*', 'space.w_None'), '_Py_TrueStruct#': ('PyObject*', 'space.w_True'), '_Py_ZeroStruct#': ('PyObject*', 'space.w_False'), + '_Py_NotImplementedStruct#': ('PyObject*', 'space.w_False'), } INIT_FUNCTIONS = [] @@ -253,7 +254,7 @@ "Unicode": "space.w_unicode", 'Bool': 'space.w_bool', 'None': 'space.type(space.w_None)', - 'NotImplemented': 'space.w_NotImplemented', + 'NotImplemented': 'space.type(space.w_NotImplemented)', }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) From jandem at codespeak.net Sat Apr 10 16:24:23 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:24:23 +0200 (CEST) Subject: [pypy-svn] r73625 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410142423.45DBC282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:24:21 2010 New Revision: 73625 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Argh forgot CTRL+S Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 16:24:21 2010 @@ -239,7 +239,7 @@ '_Py_NoneStruct#': ('PyObject*', 'space.w_None'), '_Py_TrueStruct#': ('PyObject*', 'space.w_True'), '_Py_ZeroStruct#': ('PyObject*', 'space.w_False'), - '_Py_NotImplementedStruct#': ('PyObject*', 'space.w_False'), + '_Py_NotImplementedStruct#': ('PyObject*', 'space.w_NotImplemented'), } INIT_FUNCTIONS = [] From jandem at codespeak.net Sat Apr 10 16:27:25 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:27:25 +0200 (CEST) Subject: [pypy-svn] r73626 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100410142725.C7F60282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:27:24 2010 New Revision: 73626 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: remove the declaration here Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sat Apr 10 16:27:24 2010 @@ -61,7 +61,6 @@ Py_NotImplemented is a singleton used to signal that an operation is not implemented for a given type combination. */ -PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Rich comparison opcodes */ From jandem at codespeak.net Sat Apr 10 16:31:23 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 16:31:23 +0200 (CEST) Subject: [pypy-svn] r73627 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100410143123.D8EB1282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 16:31:22 2010 New Revision: 73627 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Log: Comment out PyObject_REALLOC Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Sat Apr 10 16:31:22 2010 @@ -7,7 +7,8 @@ /* XXX use obmalloc like cpython and pypy do, otherwise we might get segfaults */ #define PyObject_MALLOC PyMem_MALLOC -#define PyObject_REALLOC PyMem_REALLOC +// we won't support this +// #define PyObject_REALLOC PyMem_REALLOC #define PyObject_FREE PyMem_FREE #define PyMem_Malloc PyMem_MALLOC From jandem at codespeak.net Sat Apr 10 17:07:56 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 17:07:56 +0200 (CEST) Subject: [pypy-svn] r73628 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100410150756.60AD7282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 17:07:54 2010 New Revision: 73628 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Pull array module from CPython, add stubs and some api functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Sat Apr 10 17:07:54 2010 @@ -19,3 +19,11 @@ raised.""" return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + at cpython_api([rffi.VOIDP], PyObject) +def PyLong_FromVoidPtr(space, p): + """Create a Python integer or long integer from the pointer p. The pointer value + can be retrieved from the resulting value using PyLong_AsVoidPtr(). + + If the integer is larger than LONG_MAX, a positive long integer is returned.""" + return space.wrap(rffi.cast(rffi.LONG, p)) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sat Apr 10 17:07:54 2010 @@ -36,6 +36,32 @@ if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: Py_DecRef(space, rffi.cast(PyObject, obj.c_ob_type)) + at cpython_api([PyTypeObjectPtr], PyObject) +def _PyObject_GC_New(space, type): + return _PyObject_New(space, type) + + at cpython_api([rffi.VOIDP_real], lltype.Void) +def PyObject_GC_Del(space, obj): + PyObject_Del(space, obj) + + at cpython_api([PyObject], lltype.Void) +def PyObject_GC_Track(space, op): + """Adds the object op to the set of container objects tracked by the + collector. The collector can run at unexpected times so objects must be + valid while being tracked. This should be called once all the fields + followed by the tp_traverse handler become valid, usually near the + end of the constructor.""" + pass + + at cpython_api([rffi.VOIDP], lltype.Void) +def PyObject_GC_UnTrack(space, op): + """Remove the object op from the set of container objects tracked by the + collector. Note that PyObject_GC_Track() can be called again on + this object to add it back to the set of tracked objects. The deallocator + (tp_dealloc handler) should call this for the object before any of + the fields used by the tp_traverse handler become invalid.""" + pass + @cpython_api([PyObject], rffi.INT_real, error=-1) def PyObject_IsTrue(space, w_obj): return space.is_true(w_obj) @@ -151,3 +177,21 @@ assert isinstance(w_type, W_TypeObject) return int(space.is_w(w_obj_type, w_type) or space.is_true(space.issubtype(w_obj_type, w_type))) + + at cpython_api([PyObject], PyObject) +def PyObject_SelfIter(space, ref): + """Undocumented function, this is wat CPython does.""" + Py_IncRef(space, ref) + return ref + + at cpython_api([PyObject, PyObject], PyObject) +def PyObject_GenericGetAttr(space, w_obj, w_name): + """Generic attribute getter function that is meant to be put into a type + object's tp_getattro slot. It looks for a descriptor in the dictionary + of classes in the object's MRO as well as an attribute in the object's + __dict__ (if present). As outlined in descriptors, data + descriptors take preference over instance attributes, while non-data + descriptors don't. Otherwise, an AttributeError is raised.""" + from pypy.objspace.descroperation import object_getattribute + w_descr = object_getattribute(space) + return space.get_and_call_function(w_descr, w_obj, w_name) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Sat Apr 10 17:07:54 2010 @@ -2,12 +2,34 @@ from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL from pypy.rpython.lltypesystem import rffi, lltype - at cpython_api([PyObject, PyObject], PyObject) -def PyObject_GenericGetAttr(space, o, name): - """Generic attribute getter function that is meant to be put into a type - object's tp_getattro slot. It looks for a descriptor in the dictionary - of classes in the object's MRO as well as an attribute in the object's - __dict__ (if present). As outlined in descriptors, data - descriptors take preference over instance attributes, while non-data - descriptors don't. Otherwise, an AttributeError is raised.""" + at cpython_api([PyObject], rffi.VOIDP, error=CANNOT_FAIL) #XXX +def PyFile_AsFile(space, p): + """Return the file object associated with p as a FILE*. + + If the caller will ever use the returned FILE* object while + the GIL is released it must also call the PyFile_IncUseCount() and + PyFile_DecUseCount() functions described below as appropriate.""" + raise NotImplementedError + + at cpython_api([PyObject], PyObject) +def PyObject_GetIter(space, o): + """This is equivalent to the Python expression iter(o). It returns a new + iterator for the object argument, or the object itself if the object is already + an iterator. Raises TypeError and returns NULL if the object cannot be + iterated.""" + raise NotImplementedError + + at cpython_api([PyObject], PyObject) +def PyIter_Next(space, o): + """Return the next value from the iteration o. If the object is an iterator, + this retrieves the next value from the iteration, and returns NULL with no + exception set if there are no remaining items. If the object is not an + iterator, TypeError is raised, or if there is an error in retrieving the + item, returns NULL and passes along the exception.""" + raise NotImplementedError + + at cpython_api([rffi.ULONG], PyObject) +def PyLong_FromUnsignedLong(space, v): + """Return a new PyLongObject object from a C unsigned long, or + NULL on failure.""" raise NotImplementedError Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c Sat Apr 10 17:07:54 2010 @@ -0,0 +1,2242 @@ +/* Array object implementation */ + +/* An array is a uniform list -- all items have the same type. + The item type is restricted to simple C types like int or float */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" + +#ifdef STDC_HEADERS +#include +#else /* !STDC_HEADERS */ +#ifdef HAVE_SYS_TYPES_H +#include /* For size_t */ +#endif /* HAVE_SYS_TYPES_H */ +#endif /* !STDC_HEADERS */ + +struct arrayobject; /* Forward */ + +/* All possible arraydescr values are defined in the vector "descriptors" + * below. That's defined later because the appropriate get and set + * functions aren't visible yet. + */ +struct arraydescr { + int typecode; + int itemsize; + PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); + int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); +}; + +typedef struct arrayobject { + PyObject_VAR_HEAD + char *ob_item; + Py_ssize_t allocated; + struct arraydescr *ob_descr; + PyObject *weakreflist; /* List of weak references */ +} arrayobject; + +static PyTypeObject Arraytype; + +#define array_Check(op) PyObject_TypeCheck(op, &Arraytype) +#define array_CheckExact(op) (Py_TYPE(op) == &Arraytype) + +static int +array_resize(arrayobject *self, Py_ssize_t newsize) +{ + char *items; + size_t _new_size; + + /* Bypass realloc() when a previous overallocation is large enough + to accommodate the newsize. If the newsize is 16 smaller than the + current size, then proceed with the realloc() to shrink the list. + */ + + if (self->allocated >= newsize && + Py_SIZE(self) < newsize + 16 && + self->ob_item != NULL) { + Py_SIZE(self) = newsize; + return 0; + } + + /* This over-allocates proportional to the array size, making room + * for additional growth. The over-allocation is mild, but is + * enough to give linear-time amortized behavior over a long + * sequence of appends() in the presence of a poorly-performing + * system realloc(). + * The growth pattern is: 0, 4, 8, 16, 25, 34, 46, 56, 67, 79, ... + * Note, the pattern starts out the same as for lists but then + * grows at a smaller rate so that larger arrays only overallocate + * by about 1/16th -- this is done because arrays are presumed to be more + * memory critical. + */ + + _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; + items = self->ob_item; + /* XXX The following multiplication and division does not optimize away + like it does for lists since the size is not known at compile time */ + if (_new_size <= ((~(size_t)0) / self->ob_descr->itemsize)) + PyMem_RESIZE(items, char, (_new_size * self->ob_descr->itemsize)); + else + items = NULL; + if (items == NULL) { + PyErr_NoMemory(); + return -1; + } + self->ob_item = items; + Py_SIZE(self) = newsize; + self->allocated = _new_size; + return 0; +} + +/**************************************************************************** +Get and Set functions for each type. +A Get function takes an arrayobject* and an integer index, returning the +array value at that index wrapped in an appropriate PyObject*. +A Set function takes an arrayobject, integer index, and PyObject*; sets +the array value at that index to the raw C data extracted from the PyObject*, +and returns 0 if successful, else nonzero on failure (PyObject* not of an +appropriate type or value). +Note that the basic Get and Set functions do NOT check that the index is +in bounds; that's the responsibility of the caller. +****************************************************************************/ + +static PyObject * +c_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyString_FromStringAndSize(&((char *)ap->ob_item)[i], 1); +} + +static int +c_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + char x; + if (!PyArg_Parse(v, "c;array item must be char", &x)) + return -1; + if (i >= 0) + ((char *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +b_getitem(arrayobject *ap, Py_ssize_t i) +{ + long x = ((char *)ap->ob_item)[i]; + if (x >= 128) + x -= 256; + return PyInt_FromLong(x); +} + +static int +b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + short x; + /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore + must use the next size up that is signed ('h') and manually do + the overflow checking */ + if (!PyArg_Parse(v, "h;array item must be integer", &x)) + return -1; + else if (x < -128) { + PyErr_SetString(PyExc_OverflowError, + "signed char is less than minimum"); + return -1; + } + else if (x > 127) { + PyErr_SetString(PyExc_OverflowError, + "signed char is greater than maximum"); + return -1; + } + if (i >= 0) + ((char *)ap->ob_item)[i] = (char)x; + return 0; +} + +static PyObject * +BB_getitem(arrayobject *ap, Py_ssize_t i) +{ + long x = ((unsigned char *)ap->ob_item)[i]; + return PyInt_FromLong(x); +} + +static int +BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + unsigned char x; + /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ + if (!PyArg_Parse(v, "b;array item must be integer", &x)) + return -1; + if (i >= 0) + ((char *)ap->ob_item)[i] = x; + return 0; +} + +#ifdef Py_USING_UNICODE +static PyObject * +u_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyUnicode_FromUnicode(&((Py_UNICODE *) ap->ob_item)[i], 1); +} + +static int +u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + Py_UNICODE *p; + Py_ssize_t len; + + if (!PyArg_Parse(v, "u#;array item must be unicode character", &p, &len)) + return -1; + if (len != 1) { + PyErr_SetString(PyExc_TypeError, + "array item must be unicode character"); + return -1; + } + if (i >= 0) + ((Py_UNICODE *)ap->ob_item)[i] = p[0]; + return 0; +} +#endif + +static PyObject * +h_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyInt_FromLong((long) ((short *)ap->ob_item)[i]); +} + +static int +h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + short x; + /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ + if (!PyArg_Parse(v, "h;array item must be integer", &x)) + return -1; + if (i >= 0) + ((short *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +HH_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyInt_FromLong((long) ((unsigned short *)ap->ob_item)[i]); +} + +static int +HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + int x; + /* PyArg_Parse's 'h' formatter is for a signed short, therefore + must use the next size up and manually do the overflow checking */ + if (!PyArg_Parse(v, "i;array item must be integer", &x)) + return -1; + else if (x < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned short is less than minimum"); + return -1; + } + else if (x > USHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned short is greater than maximum"); + return -1; + } + if (i >= 0) + ((short *)ap->ob_item)[i] = (short)x; + return 0; +} + +static PyObject * +i_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyInt_FromLong((long) ((int *)ap->ob_item)[i]); +} + +static int +i_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + int x; + /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ + if (!PyArg_Parse(v, "i;array item must be integer", &x)) + return -1; + if (i >= 0) + ((int *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +II_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyLong_FromUnsignedLong( + (unsigned long) ((unsigned int *)ap->ob_item)[i]); +} + +static int +II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + unsigned long x; + if (PyLong_Check(v)) { + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long) -1 && PyErr_Occurred()) + return -1; + } + else { + long y; + if (!PyArg_Parse(v, "l;array item must be integer", &y)) + return -1; + if (y < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned int is less than minimum"); + return -1; + } + x = (unsigned long)y; + + } + if (x > UINT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned int is greater than maximum"); + return -1; + } + + if (i >= 0) + ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + return 0; +} + +static PyObject * +l_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyInt_FromLong(((long *)ap->ob_item)[i]); +} + +static int +l_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + long x; + if (!PyArg_Parse(v, "l;array item must be integer", &x)) + return -1; + if (i >= 0) + ((long *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +LL_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyLong_FromUnsignedLong(((unsigned long *)ap->ob_item)[i]); +} + +static int +LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + unsigned long x; + if (PyLong_Check(v)) { + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long) -1 && PyErr_Occurred()) + return -1; + } + else { + long y; + if (!PyArg_Parse(v, "l;array item must be integer", &y)) + return -1; + if (y < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned long is less than minimum"); + return -1; + } + x = (unsigned long)y; + + } + if (x > ULONG_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned long is greater than maximum"); + return -1; + } + + if (i >= 0) + ((unsigned long *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +f_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyFloat_FromDouble((double) ((float *)ap->ob_item)[i]); +} + +static int +f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + float x; + if (!PyArg_Parse(v, "f;array item must be float", &x)) + return -1; + if (i >= 0) + ((float *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +d_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyFloat_FromDouble(((double *)ap->ob_item)[i]); +} + +static int +d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + double x; + if (!PyArg_Parse(v, "d;array item must be float", &x)) + return -1; + if (i >= 0) + ((double *)ap->ob_item)[i] = x; + return 0; +} + +/* Description of types */ +static struct arraydescr descriptors[] = { + {'c', sizeof(char), c_getitem, c_setitem}, + {'b', sizeof(char), b_getitem, b_setitem}, + {'B', sizeof(char), BB_getitem, BB_setitem}, +#ifdef Py_USING_UNICODE + {'u', sizeof(Py_UNICODE), u_getitem, u_setitem}, +#endif + {'h', sizeof(short), h_getitem, h_setitem}, + {'H', sizeof(short), HH_getitem, HH_setitem}, + {'i', sizeof(int), i_getitem, i_setitem}, + {'I', sizeof(int), II_getitem, II_setitem}, + {'l', sizeof(long), l_getitem, l_setitem}, + {'L', sizeof(long), LL_getitem, LL_setitem}, + {'f', sizeof(float), f_getitem, f_setitem}, + {'d', sizeof(double), d_getitem, d_setitem}, + {'\0', 0, 0, 0} /* Sentinel */ +}; + +/**************************************************************************** +Implementations of array object methods. +****************************************************************************/ + +static PyObject * +newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr) +{ + arrayobject *op; + size_t nbytes; + + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } + + nbytes = size * descr->itemsize; + /* Check for overflow */ + if (nbytes / descr->itemsize != (size_t)size) { + return PyErr_NoMemory(); + } + op = (arrayobject *) type->tp_alloc(type, 0); + if (op == NULL) { + return NULL; + } + op->ob_descr = descr; + op->allocated = size; + op->weakreflist = NULL; + Py_SIZE(op) = size; + if (size <= 0) { + op->ob_item = NULL; + } + else { + op->ob_item = PyMem_NEW(char, nbytes); + if (op->ob_item == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + } + return (PyObject *) op; +} + +static PyObject * +getarrayitem(PyObject *op, Py_ssize_t i) +{ + register arrayobject *ap; + assert(array_Check(op)); + ap = (arrayobject *)op; + assert(i>=0 && iob_descr->getitem)(ap, i); +} + +static int +ins1(arrayobject *self, Py_ssize_t where, PyObject *v) +{ + char *items; + Py_ssize_t n = Py_SIZE(self); + if (v == NULL) { + PyErr_BadInternalCall(); + return -1; + } + if ((*self->ob_descr->setitem)(self, -1, v) < 0) + return -1; + + if (array_resize(self, n+1) == -1) + return -1; + items = self->ob_item; + if (where < 0) { + where += n; + if (where < 0) + where = 0; + } + if (where > n) + where = n; + /* appends don't need to call memmove() */ + if (where != n) + memmove(items + (where+1)*self->ob_descr->itemsize, + items + where*self->ob_descr->itemsize, + (n-where)*self->ob_descr->itemsize); + return (*self->ob_descr->setitem)(self, where, v); +} + +/* Methods */ + +static void +array_dealloc(arrayobject *op) +{ + if (op->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) op); + if (op->ob_item != NULL) + PyMem_DEL(op->ob_item); + Py_TYPE(op)->tp_free((PyObject *)op); +} + +static PyObject * +array_richcompare(PyObject *v, PyObject *w, int op) +{ + arrayobject *va, *wa; + PyObject *vi = NULL; + PyObject *wi = NULL; + Py_ssize_t i, k; + PyObject *res; + + if (!array_Check(v) || !array_Check(w)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + va = (arrayobject *)v; + wa = (arrayobject *)w; + + if (Py_SIZE(va) != Py_SIZE(wa) && (op == Py_EQ || op == Py_NE)) { + /* Shortcut: if the lengths differ, the arrays differ */ + if (op == Py_EQ) + res = Py_False; + else + res = Py_True; + Py_INCREF(res); + return res; + } + + /* Search for the first index where items are different */ + k = 1; + for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { + vi = getarrayitem(v, i); + wi = getarrayitem(w, i); + if (vi == NULL || wi == NULL) { + Py_XDECREF(vi); + Py_XDECREF(wi); + return NULL; + } + k = PyObject_RichCompareBool(vi, wi, Py_EQ); + if (k == 0) + break; /* Keeping vi and wi alive! */ + Py_DECREF(vi); + Py_DECREF(wi); + if (k < 0) + return NULL; + } + + if (k) { + /* No more items to compare -- compare sizes */ + Py_ssize_t vs = Py_SIZE(va); + Py_ssize_t ws = Py_SIZE(wa); + int cmp; + switch (op) { + case Py_LT: cmp = vs < ws; break; + case Py_LE: cmp = vs <= ws; break; + case Py_EQ: cmp = vs == ws; break; + case Py_NE: cmp = vs != ws; break; + case Py_GT: cmp = vs > ws; break; + case Py_GE: cmp = vs >= ws; break; + default: return NULL; /* cannot happen */ + } + if (cmp) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + } + + /* We have an item that differs. First, shortcuts for EQ/NE */ + if (op == Py_EQ) { + Py_INCREF(Py_False); + res = Py_False; + } + else if (op == Py_NE) { + Py_INCREF(Py_True); + res = Py_True; + } + else { + /* Compare the final item again using the proper operator */ + res = PyObject_RichCompare(vi, wi, op); + } + Py_DECREF(vi); + Py_DECREF(wi); + return res; +} + +static Py_ssize_t +array_length(arrayobject *a) +{ + return Py_SIZE(a); +} + +static PyObject * +array_item(arrayobject *a, Py_ssize_t i) +{ + if (i < 0 || i >= Py_SIZE(a)) { + PyErr_SetString(PyExc_IndexError, "array index out of range"); + return NULL; + } + return getarrayitem((PyObject *)a, i); +} + +static PyObject * +array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +{ + arrayobject *np; + if (ilow < 0) + ilow = 0; + else if (ilow > Py_SIZE(a)) + ilow = Py_SIZE(a); + if (ihigh < 0) + ihigh = 0; + if (ihigh < ilow) + ihigh = ilow; + else if (ihigh > Py_SIZE(a)) + ihigh = Py_SIZE(a); + np = (arrayobject *) newarrayobject(&Arraytype, ihigh - ilow, a->ob_descr); + if (np == NULL) + return NULL; + memcpy(np->ob_item, a->ob_item + ilow * a->ob_descr->itemsize, + (ihigh-ilow) * a->ob_descr->itemsize); + return (PyObject *)np; +} + +static PyObject * +array_copy(arrayobject *a, PyObject *unused) +{ + return array_slice(a, 0, Py_SIZE(a)); +} + +PyDoc_STRVAR(copy_doc, +"copy(array)\n\ +\n\ + Return a copy of the array."); + +static PyObject * +array_concat(arrayobject *a, PyObject *bb) +{ + Py_ssize_t size; + arrayobject *np; + if (!array_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only append array (not \"%.200s\") to array", + Py_TYPE(bb)->tp_name); + return NULL; + } +#define b ((arrayobject *)bb) + if (a->ob_descr != b->ob_descr) { + PyErr_BadArgument(); + return NULL; + } + if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) { + return PyErr_NoMemory(); + } + size = Py_SIZE(a) + Py_SIZE(b); + np = (arrayobject *) newarrayobject(&Arraytype, size, a->ob_descr); + if (np == NULL) { + return NULL; + } + memcpy(np->ob_item, a->ob_item, Py_SIZE(a)*a->ob_descr->itemsize); + memcpy(np->ob_item + Py_SIZE(a)*a->ob_descr->itemsize, + b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); + return (PyObject *)np; +#undef b +} + +static PyObject * +array_repeat(arrayobject *a, Py_ssize_t n) +{ + Py_ssize_t i; + Py_ssize_t size; + arrayobject *np; + char *p; + Py_ssize_t nbytes; + if (n < 0) + n = 0; + if ((Py_SIZE(a) != 0) && (n > PY_SSIZE_T_MAX / Py_SIZE(a))) { + return PyErr_NoMemory(); + } + size = Py_SIZE(a) * n; + np = (arrayobject *) newarrayobject(&Arraytype, size, a->ob_descr); + if (np == NULL) + return NULL; + p = np->ob_item; + nbytes = Py_SIZE(a) * a->ob_descr->itemsize; + for (i = 0; i < n; i++) { + memcpy(p, a->ob_item, nbytes); + p += nbytes; + } + return (PyObject *) np; +} + +static int +array_ass_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +{ + char *item; + Py_ssize_t n; /* Size of replacement array */ + Py_ssize_t d; /* Change in size */ +#define b ((arrayobject *)v) + if (v == NULL) + n = 0; + else if (array_Check(v)) { + n = Py_SIZE(b); + if (a == b) { + /* Special case "a[i:j] = a" -- copy b first */ + int ret; + v = array_slice(b, 0, n); + if (!v) + return -1; + ret = array_ass_slice(a, ilow, ihigh, v); + Py_DECREF(v); + return ret; + } + if (b->ob_descr != a->ob_descr) { + PyErr_BadArgument(); + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "can only assign array (not \"%.200s\") to array slice", + Py_TYPE(v)->tp_name); + return -1; + } + if (ilow < 0) + ilow = 0; + else if (ilow > Py_SIZE(a)) + ilow = Py_SIZE(a); + if (ihigh < 0) + ihigh = 0; + if (ihigh < ilow) + ihigh = ilow; + else if (ihigh > Py_SIZE(a)) + ihigh = Py_SIZE(a); + item = a->ob_item; + d = n - (ihigh-ilow); + if (d < 0) { /* Delete -d items */ + memmove(item + (ihigh+d)*a->ob_descr->itemsize, + item + ihigh*a->ob_descr->itemsize, + (Py_SIZE(a)-ihigh)*a->ob_descr->itemsize); + Py_SIZE(a) += d; + PyMem_RESIZE(item, char, Py_SIZE(a)*a->ob_descr->itemsize); + /* Can't fail */ + a->ob_item = item; + a->allocated = Py_SIZE(a); + } + else if (d > 0) { /* Insert d items */ + PyMem_RESIZE(item, char, + (Py_SIZE(a) + d)*a->ob_descr->itemsize); + if (item == NULL) { + PyErr_NoMemory(); + return -1; + } + memmove(item + (ihigh+d)*a->ob_descr->itemsize, + item + ihigh*a->ob_descr->itemsize, + (Py_SIZE(a)-ihigh)*a->ob_descr->itemsize); + a->ob_item = item; + Py_SIZE(a) += d; + a->allocated = Py_SIZE(a); + } + if (n > 0) + memcpy(item + ilow*a->ob_descr->itemsize, b->ob_item, + n*b->ob_descr->itemsize); + return 0; +#undef b +} + +static int +array_ass_item(arrayobject *a, Py_ssize_t i, PyObject *v) +{ + if (i < 0 || i >= Py_SIZE(a)) { + PyErr_SetString(PyExc_IndexError, + "array assignment index out of range"); + return -1; + } + if (v == NULL) + return array_ass_slice(a, i, i+1, v); + return (*a->ob_descr->setitem)(a, i, v); +} + +static int +setarrayitem(PyObject *a, Py_ssize_t i, PyObject *v) +{ + assert(array_Check(a)); + return array_ass_item((arrayobject *)a, i, v); +} + +static int +array_iter_extend(arrayobject *self, PyObject *bb) +{ + PyObject *it, *v; + + it = PyObject_GetIter(bb); + if (it == NULL) + return -1; + + while ((v = PyIter_Next(it)) != NULL) { + if (ins1(self, (int) Py_SIZE(self), v) != 0) { + Py_DECREF(v); + Py_DECREF(it); + return -1; + } + Py_DECREF(v); + } + Py_DECREF(it); + if (PyErr_Occurred()) + return -1; + return 0; +} + +static int +array_do_extend(arrayobject *self, PyObject *bb) +{ + Py_ssize_t size; + char *old_item; + + if (!array_Check(bb)) + return array_iter_extend(self, bb); +#define b ((arrayobject *)bb) + if (self->ob_descr != b->ob_descr) { + PyErr_SetString(PyExc_TypeError, + "can only extend with array of same kind"); + return -1; + } + if ((Py_SIZE(self) > PY_SSIZE_T_MAX - Py_SIZE(b)) || + ((Py_SIZE(self) + Py_SIZE(b)) > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { + PyErr_NoMemory(); + return -1; + } + size = Py_SIZE(self) + Py_SIZE(b); + old_item = self->ob_item; + PyMem_RESIZE(self->ob_item, char, size*self->ob_descr->itemsize); + if (self->ob_item == NULL) { + self->ob_item = old_item; + PyErr_NoMemory(); + return -1; + } + memcpy(self->ob_item + Py_SIZE(self)*self->ob_descr->itemsize, + b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); + Py_SIZE(self) = size; + self->allocated = size; + + return 0; +#undef b +} + +static PyObject * +array_inplace_concat(arrayobject *self, PyObject *bb) +{ + if (!array_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only extend array with array (not \"%.200s\")", + Py_TYPE(bb)->tp_name); + return NULL; + } + if (array_do_extend(self, bb) == -1) + return NULL; + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +array_inplace_repeat(arrayobject *self, Py_ssize_t n) +{ + char *items, *p; + Py_ssize_t size, i; + + if (Py_SIZE(self) > 0) { + if (n < 0) + n = 0; + items = self->ob_item; + if ((self->ob_descr->itemsize != 0) && + (Py_SIZE(self) > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { + return PyErr_NoMemory(); + } + size = Py_SIZE(self) * self->ob_descr->itemsize; + if (n == 0) { + PyMem_FREE(items); + self->ob_item = NULL; + Py_SIZE(self) = 0; + self->allocated = 0; + } + else { + if (size > PY_SSIZE_T_MAX / n) { + return PyErr_NoMemory(); + } + PyMem_RESIZE(items, char, n * size); + if (items == NULL) + return PyErr_NoMemory(); + p = items; + for (i = 1; i < n; i++) { + p += size; + memcpy(p, items, size); + } + self->ob_item = items; + Py_SIZE(self) *= n; + self->allocated = Py_SIZE(self); + } + } + Py_INCREF(self); + return (PyObject *)self; +} + + +static PyObject * +ins(arrayobject *self, Py_ssize_t where, PyObject *v) +{ + if (ins1(self, where, v) != 0) + return NULL; + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +array_count(arrayobject *self, PyObject *v) +{ + Py_ssize_t count = 0; + Py_ssize_t i; + + for (i = 0; i < Py_SIZE(self); i++) { + PyObject *selfi = getarrayitem((PyObject *)self, i); + int cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); + Py_DECREF(selfi); + if (cmp > 0) + count++; + else if (cmp < 0) + return NULL; + } + return PyInt_FromSsize_t(count); +} + +PyDoc_STRVAR(count_doc, +"count(x)\n\ +\n\ +Return number of occurrences of x in the array."); + +static PyObject * +array_index(arrayobject *self, PyObject *v) +{ + Py_ssize_t i; + + for (i = 0; i < Py_SIZE(self); i++) { + PyObject *selfi = getarrayitem((PyObject *)self, i); + int cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); + Py_DECREF(selfi); + if (cmp > 0) { + return PyInt_FromLong((long)i); + } + else if (cmp < 0) + return NULL; + } + PyErr_SetString(PyExc_ValueError, "array.index(x): x not in list"); + return NULL; +} + +PyDoc_STRVAR(index_doc, +"index(x)\n\ +\n\ +Return index of first occurrence of x in the array."); + +static int +array_contains(arrayobject *self, PyObject *v) +{ + Py_ssize_t i; + int cmp; + + for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) { + PyObject *selfi = getarrayitem((PyObject *)self, i); + cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); + Py_DECREF(selfi); + } + return cmp; +} + +static PyObject * +array_remove(arrayobject *self, PyObject *v) +{ + int i; + + for (i = 0; i < Py_SIZE(self); i++) { + PyObject *selfi = getarrayitem((PyObject *)self,i); + int cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); + Py_DECREF(selfi); + if (cmp > 0) { + if (array_ass_slice(self, i, i+1, + (PyObject *)NULL) != 0) + return NULL; + Py_INCREF(Py_None); + return Py_None; + } + else if (cmp < 0) + return NULL; + } + PyErr_SetString(PyExc_ValueError, "array.remove(x): x not in list"); + return NULL; +} + +PyDoc_STRVAR(remove_doc, +"remove(x)\n\ +\n\ +Remove the first occurrence of x in the array."); + +static PyObject * +array_pop(arrayobject *self, PyObject *args) +{ + Py_ssize_t i = -1; + PyObject *v; + if (!PyArg_ParseTuple(args, "|n:pop", &i)) + return NULL; + if (Py_SIZE(self) == 0) { + /* Special-case most common failure cause */ + PyErr_SetString(PyExc_IndexError, "pop from empty array"); + return NULL; + } + if (i < 0) + i += Py_SIZE(self); + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "pop index out of range"); + return NULL; + } + v = getarrayitem((PyObject *)self,i); + if (array_ass_slice(self, i, i+1, (PyObject *)NULL) != 0) { + Py_DECREF(v); + return NULL; + } + return v; +} + +PyDoc_STRVAR(pop_doc, +"pop([i])\n\ +\n\ +Return the i-th element and delete it from the array. i defaults to -1."); + +static PyObject * +array_extend(arrayobject *self, PyObject *bb) +{ + if (array_do_extend(self, bb) == -1) + return NULL; + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(extend_doc, +"extend(array or iterable)\n\ +\n\ + Append items to the end of the array."); + +static PyObject * +array_insert(arrayobject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *v; + if (!PyArg_ParseTuple(args, "nO:insert", &i, &v)) + return NULL; + return ins(self, i, v); +} + +PyDoc_STRVAR(insert_doc, +"insert(i,x)\n\ +\n\ +Insert a new item x into the array before position i."); + + +static PyObject * +array_buffer_info(arrayobject *self, PyObject *unused) +{ + PyObject* retval = NULL; + retval = PyTuple_New(2); + if (!retval) + return NULL; + + PyTuple_SET_ITEM(retval, 0, PyLong_FromVoidPtr(self->ob_item)); + PyTuple_SET_ITEM(retval, 1, PyInt_FromLong((long)(Py_SIZE(self)))); + + return retval; +} + +PyDoc_STRVAR(buffer_info_doc, +"buffer_info() -> (address, length)\n\ +\n\ +Return a tuple (address, length) giving the current memory address and\n\ +the length in items of the buffer used to hold array's contents\n\ +The length should be multiplied by the itemsize attribute to calculate\n\ +the buffer length in bytes."); + + +static PyObject * +array_append(arrayobject *self, PyObject *v) +{ + return ins(self, (int) Py_SIZE(self), v); +} + +PyDoc_STRVAR(append_doc, +"append(x)\n\ +\n\ +Append new value x to the end of the array."); + + +static PyObject * +array_byteswap(arrayobject *self, PyObject *unused) +{ + char *p; + Py_ssize_t i; + + switch (self->ob_descr->itemsize) { + case 1: + break; + case 2: + for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 2) { + char p0 = p[0]; + p[0] = p[1]; + p[1] = p0; + } + break; + case 4: + for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 4) { + char p0 = p[0]; + char p1 = p[1]; + p[0] = p[3]; + p[1] = p[2]; + p[2] = p1; + p[3] = p0; + } + break; + case 8: + for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { + char p0 = p[0]; + char p1 = p[1]; + char p2 = p[2]; + char p3 = p[3]; + p[0] = p[7]; + p[1] = p[6]; + p[2] = p[5]; + p[3] = p[4]; + p[4] = p3; + p[5] = p2; + p[6] = p1; + p[7] = p0; + } + break; + default: + PyErr_SetString(PyExc_RuntimeError, + "don't know how to byteswap this array type"); + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(byteswap_doc, +"byteswap()\n\ +\n\ +Byteswap all items of the array. If the items in the array are not 1, 2,\n\ +4, or 8 bytes in size, RuntimeError is raised."); + +static PyObject * +array_reduce(arrayobject *array) +{ + PyObject *dict, *result; + + dict = PyObject_GetAttrString((PyObject *)array, "__dict__"); + if (dict == NULL) { + PyErr_Clear(); + dict = Py_None; + Py_INCREF(dict); + } + if (Py_SIZE(array) > 0) { + if (array->ob_descr->itemsize + > PY_SSIZE_T_MAX / array->ob_size) { + return PyErr_NoMemory(); + } + result = Py_BuildValue("O(cs#)O", + Py_TYPE(array), + array->ob_descr->typecode, + array->ob_item, + Py_SIZE(array) * array->ob_descr->itemsize, + dict); + } else { + result = Py_BuildValue("O(c)O", + Py_TYPE(array), + array->ob_descr->typecode, + dict); + } + Py_DECREF(dict); + return result; +} + +PyDoc_STRVAR(array_doc, "Return state information for pickling."); + +static PyObject * +array_reverse(arrayobject *self, PyObject *unused) +{ + register Py_ssize_t itemsize = self->ob_descr->itemsize; + register char *p, *q; + /* little buffer to hold items while swapping */ + char tmp[256]; /* 8 is probably enough -- but why skimp */ + assert((size_t)itemsize <= sizeof(tmp)); + + if (Py_SIZE(self) > 1) { + for (p = self->ob_item, + q = self->ob_item + (Py_SIZE(self) - 1)*itemsize; + p < q; + p += itemsize, q -= itemsize) { + /* memory areas guaranteed disjoint, so memcpy + * is safe (& memmove may be slower). + */ + memcpy(tmp, p, itemsize); + memcpy(p, q, itemsize); + memcpy(q, tmp, itemsize); + } + } + + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(reverse_doc, +"reverse()\n\ +\n\ +Reverse the order of the items in the array."); + +static PyObject * +array_fromfile(arrayobject *self, PyObject *args) +{ + PyObject *f; + Py_ssize_t n; + FILE *fp; + if (!PyArg_ParseTuple(args, "On:fromfile", &f, &n)) + return NULL; + fp = PyFile_AsFile(f); + if (fp == NULL) { + PyErr_SetString(PyExc_TypeError, "arg1 must be open file"); + return NULL; + } + if (n > 0) { + char *item = self->ob_item; + Py_ssize_t itemsize = self->ob_descr->itemsize; + size_t nread; + Py_ssize_t newlength; + size_t newbytes; + /* Be careful here about overflow */ + if ((newlength = Py_SIZE(self) + n) <= 0 || + (newbytes = newlength * itemsize) / itemsize != + (size_t)newlength) + goto nomem; + PyMem_RESIZE(item, char, newbytes); + if (item == NULL) { + nomem: + PyErr_NoMemory(); + return NULL; + } + self->ob_item = item; + Py_SIZE(self) += n; + self->allocated = Py_SIZE(self); + nread = fread(item + (Py_SIZE(self) - n) * itemsize, + itemsize, n, fp); + if (nread < (size_t)n) { + Py_SIZE(self) -= (n - nread); + PyMem_RESIZE(item, char, Py_SIZE(self)*itemsize); + self->ob_item = item; + self->allocated = Py_SIZE(self); + PyErr_SetString(PyExc_EOFError, + "not enough items in file"); + return NULL; + } + } + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(fromfile_doc, +"fromfile(f, n)\n\ +\n\ +Read n objects from the file object f and append them to the end of the\n\ +array. Also called as read."); + + +static PyObject * +array_fromfile_as_read(arrayobject *self, PyObject *args) +{ + if (PyErr_WarnPy3k("array.read() not supported in 3.x; " + "use array.fromfile()", 1) < 0) + return NULL; + return array_fromfile(self, args); +} + + +static PyObject * +array_tofile(arrayobject *self, PyObject *f) +{ + FILE *fp; + + fp = PyFile_AsFile(f); + if (fp == NULL) { + PyErr_SetString(PyExc_TypeError, "arg must be open file"); + return NULL; + } + if (self->ob_size > 0) { + if (fwrite(self->ob_item, self->ob_descr->itemsize, + self->ob_size, fp) != (size_t)self->ob_size) { + PyErr_SetFromErrno(PyExc_IOError); + clearerr(fp); + return NULL; + } + } + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(tofile_doc, +"tofile(f)\n\ +\n\ +Write all items (as machine values) to the file object f. Also called as\n\ +write."); + + +static PyObject * +array_tofile_as_write(arrayobject *self, PyObject *f) +{ + if (PyErr_WarnPy3k("array.write() not supported in 3.x; " + "use array.tofile()", 1) < 0) + return NULL; + return array_tofile(self, f); +} + + +static PyObject * +array_fromlist(arrayobject *self, PyObject *list) +{ + Py_ssize_t n; + Py_ssize_t itemsize = self->ob_descr->itemsize; + + if (!PyList_Check(list)) { + PyErr_SetString(PyExc_TypeError, "arg must be list"); + return NULL; + } + n = PyList_Size(list); + if (n > 0) { + char *item = self->ob_item; + Py_ssize_t i; + PyMem_RESIZE(item, char, (Py_SIZE(self) + n) * itemsize); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + self->ob_item = item; + Py_SIZE(self) += n; + self->allocated = Py_SIZE(self); + for (i = 0; i < n; i++) { + PyObject *v = PyList_GetItem(list, i); + if ((*self->ob_descr->setitem)(self, + Py_SIZE(self) - n + i, v) != 0) { + Py_SIZE(self) -= n; + if (itemsize && (self->ob_size > PY_SSIZE_T_MAX / itemsize)) { + return PyErr_NoMemory(); + } + PyMem_RESIZE(item, char, + Py_SIZE(self) * itemsize); + self->ob_item = item; + self->allocated = Py_SIZE(self); + return NULL; + } + } + } + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(fromlist_doc, +"fromlist(list)\n\ +\n\ +Append items to array from list."); + + +static PyObject * +array_tolist(arrayobject *self, PyObject *unused) +{ + PyObject *list = PyList_New(Py_SIZE(self)); + Py_ssize_t i; + + if (list == NULL) + return NULL; + for (i = 0; i < Py_SIZE(self); i++) { + PyObject *v = getarrayitem((PyObject *)self, i); + if (v == NULL) { + Py_DECREF(list); + return NULL; + } + PyList_SetItem(list, i, v); + } + return list; +} + +PyDoc_STRVAR(tolist_doc, +"tolist() -> list\n\ +\n\ +Convert array to an ordinary list with the same items."); + + +static PyObject * +array_fromstring(arrayobject *self, PyObject *args) +{ + char *str; + Py_ssize_t n; + int itemsize = self->ob_descr->itemsize; + if (!PyArg_ParseTuple(args, "s#:fromstring", &str, &n)) + return NULL; + if (n % itemsize != 0) { + PyErr_SetString(PyExc_ValueError, + "string length not a multiple of item size"); + return NULL; + } + n = n / itemsize; + if (n > 0) { + char *item = self->ob_item; + if ((n > PY_SSIZE_T_MAX - Py_SIZE(self)) || + ((Py_SIZE(self) + n) > PY_SSIZE_T_MAX / itemsize)) { + return PyErr_NoMemory(); + } + PyMem_RESIZE(item, char, (Py_SIZE(self) + n) * itemsize); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + self->ob_item = item; + Py_SIZE(self) += n; + self->allocated = Py_SIZE(self); + memcpy(item + (Py_SIZE(self) - n) * itemsize, + str, itemsize*n); + } + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(fromstring_doc, +"fromstring(string)\n\ +\n\ +Appends items from the string, interpreting it as an array of machine\n\ +values,as if it had been read from a file using the fromfile() method)."); + + +static PyObject * +array_tostring(arrayobject *self, PyObject *unused) +{ + if (self->ob_size <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { + return PyString_FromStringAndSize(self->ob_item, + Py_SIZE(self) * self->ob_descr->itemsize); + } else { + return PyErr_NoMemory(); + } +} + +PyDoc_STRVAR(tostring_doc, +"tostring() -> string\n\ +\n\ +Convert the array to an array of machine values and return the string\n\ +representation."); + + + +#ifdef Py_USING_UNICODE +static PyObject * +array_fromunicode(arrayobject *self, PyObject *args) +{ + Py_UNICODE *ustr; + Py_ssize_t n; + + if (!PyArg_ParseTuple(args, "u#:fromunicode", &ustr, &n)) + return NULL; + if (self->ob_descr->typecode != 'u') { + PyErr_SetString(PyExc_ValueError, + "fromunicode() may only be called on " + "type 'u' arrays"); + return NULL; + } + if (n > 0) { + Py_UNICODE *item = (Py_UNICODE *) self->ob_item; + if (Py_SIZE(self) > PY_SSIZE_T_MAX - n) { + return PyErr_NoMemory(); + } + PyMem_RESIZE(item, Py_UNICODE, Py_SIZE(self) + n); + if (item == NULL) { + PyErr_NoMemory(); + return NULL; + } + self->ob_item = (char *) item; + Py_SIZE(self) += n; + self->allocated = Py_SIZE(self); + memcpy(item + Py_SIZE(self) - n, + ustr, n * sizeof(Py_UNICODE)); + } + + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(fromunicode_doc, +"fromunicode(ustr)\n\ +\n\ +Extends this array with data from the unicode string ustr.\n\ +The array must be a type 'u' array; otherwise a ValueError\n\ +is raised. Use array.fromstring(ustr.decode(...)) to\n\ +append Unicode data to an array of some other type."); + + +static PyObject * +array_tounicode(arrayobject *self, PyObject *unused) +{ + if (self->ob_descr->typecode != 'u') { + PyErr_SetString(PyExc_ValueError, + "tounicode() may only be called on type 'u' arrays"); + return NULL; + } + return PyUnicode_FromUnicode((Py_UNICODE *) self->ob_item, Py_SIZE(self)); +} + +PyDoc_STRVAR(tounicode_doc, +"tounicode() -> unicode\n\ +\n\ +Convert the array to a unicode string. The array must be\n\ +a type 'u' array; otherwise a ValueError is raised. Use\n\ +array.tostring().decode() to obtain a unicode string from\n\ +an array of some other type."); + +#endif /* Py_USING_UNICODE */ + + +static PyObject * +array_get_typecode(arrayobject *a, void *closure) +{ + char tc = a->ob_descr->typecode; + return PyString_FromStringAndSize(&tc, 1); +} + +static PyObject * +array_get_itemsize(arrayobject *a, void *closure) +{ + return PyInt_FromLong((long)a->ob_descr->itemsize); +} + +static PyGetSetDef array_getsets [] = { + {"typecode", (getter) array_get_typecode, NULL, + "the typecode character used to create the array"}, + {"itemsize", (getter) array_get_itemsize, NULL, + "the size, in bytes, of one array item"}, + {NULL} +}; + +static PyMethodDef array_methods[] = { + {"append", (PyCFunction)array_append, METH_O, + append_doc}, + {"buffer_info", (PyCFunction)array_buffer_info, METH_NOARGS, + buffer_info_doc}, + {"byteswap", (PyCFunction)array_byteswap, METH_NOARGS, + byteswap_doc}, + {"__copy__", (PyCFunction)array_copy, METH_NOARGS, + copy_doc}, + {"count", (PyCFunction)array_count, METH_O, + count_doc}, + {"__deepcopy__",(PyCFunction)array_copy, METH_O, + copy_doc}, + {"extend", (PyCFunction)array_extend, METH_O, + extend_doc}, + {"fromfile", (PyCFunction)array_fromfile, METH_VARARGS, + fromfile_doc}, + {"fromlist", (PyCFunction)array_fromlist, METH_O, + fromlist_doc}, + {"fromstring", (PyCFunction)array_fromstring, METH_VARARGS, + fromstring_doc}, +#ifdef Py_USING_UNICODE + {"fromunicode", (PyCFunction)array_fromunicode, METH_VARARGS, + fromunicode_doc}, +#endif + {"index", (PyCFunction)array_index, METH_O, + index_doc}, + {"insert", (PyCFunction)array_insert, METH_VARARGS, + insert_doc}, + {"pop", (PyCFunction)array_pop, METH_VARARGS, + pop_doc}, + {"read", (PyCFunction)array_fromfile_as_read, METH_VARARGS, + fromfile_doc}, + {"__reduce__", (PyCFunction)array_reduce, METH_NOARGS, + array_doc}, + {"remove", (PyCFunction)array_remove, METH_O, + remove_doc}, + {"reverse", (PyCFunction)array_reverse, METH_NOARGS, + reverse_doc}, +/* {"sort", (PyCFunction)array_sort, METH_VARARGS, + sort_doc},*/ + {"tofile", (PyCFunction)array_tofile, METH_O, + tofile_doc}, + {"tolist", (PyCFunction)array_tolist, METH_NOARGS, + tolist_doc}, + {"tostring", (PyCFunction)array_tostring, METH_NOARGS, + tostring_doc}, +#ifdef Py_USING_UNICODE + {"tounicode", (PyCFunction)array_tounicode, METH_NOARGS, + tounicode_doc}, +#endif + {"write", (PyCFunction)array_tofile_as_write, METH_O, + tofile_doc}, + {NULL, NULL} /* sentinel */ +}; + +static PyObject * +array_repr(arrayobject *a) +{ + char buf[256], typecode; + PyObject *s, *t, *v = NULL; + Py_ssize_t len; + + len = Py_SIZE(a); + typecode = a->ob_descr->typecode; + if (len == 0) { + PyOS_snprintf(buf, sizeof(buf), "array('%c')", typecode); + return PyString_FromString(buf); + } + + if (typecode == 'c') + v = array_tostring(a, NULL); +#ifdef Py_USING_UNICODE + else if (typecode == 'u') + v = array_tounicode(a, NULL); +#endif + else + v = array_tolist(a, NULL); + t = PyObject_Repr(v); + Py_XDECREF(v); + + PyOS_snprintf(buf, sizeof(buf), "array('%c', ", typecode); + s = PyString_FromString(buf); + PyString_ConcatAndDel(&s, t); + PyString_ConcatAndDel(&s, PyString_FromString(")")); + return s; +} + +static PyObject* +array_subscr(arrayobject* self, PyObject* item) +{ + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i==-1 && PyErr_Occurred()) { + return NULL; + } + if (i < 0) + i += Py_SIZE(self); + return array_item(self, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength, cur, i; + PyObject* result; + arrayobject* ar; + int itemsize = self->ob_descr->itemsize; + + if (PySlice_GetIndicesEx((PySliceObject*)item, Py_SIZE(self), + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (slicelength <= 0) { + return newarrayobject(&Arraytype, 0, self->ob_descr); + } + else if (step == 1) { + PyObject *result = newarrayobject(&Arraytype, + slicelength, self->ob_descr); + if (result == NULL) + return NULL; + memcpy(((arrayobject *)result)->ob_item, + self->ob_item + start * itemsize, + slicelength * itemsize); + return result; + } + else { + result = newarrayobject(&Arraytype, slicelength, self->ob_descr); + if (!result) return NULL; + + ar = (arrayobject*)result; + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + memcpy(ar->ob_item + i*itemsize, + self->ob_item + cur*itemsize, + itemsize); + } + + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "array indices must be integers"); + return NULL; + } +} + +static int +array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) +{ + Py_ssize_t start, stop, step, slicelength, needed; + arrayobject* other; + int itemsize; + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += Py_SIZE(self); + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, + "array assignment index out of range"); + return -1; + } + if (value == NULL) { + /* Fall through to slice assignment */ + start = i; + stop = i + 1; + step = 1; + slicelength = 1; + } + else + return (*self->ob_descr->setitem)(self, i, value); + } + else if (PySlice_Check(item)) { + if (PySlice_GetIndicesEx((PySliceObject *)item, + Py_SIZE(self), &start, &stop, + &step, &slicelength) < 0) { + return -1; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "array indices must be integer"); + return -1; + } + if (value == NULL) { + other = NULL; + needed = 0; + } + else if (array_Check(value)) { + other = (arrayobject *)value; + needed = Py_SIZE(other); + if (self == other) { + /* Special case "self[i:j] = self" -- copy self first */ + int ret; + value = array_slice(other, 0, needed); + if (value == NULL) + return -1; + ret = array_ass_subscr(self, item, value); + Py_DECREF(value); + return ret; + } + if (other->ob_descr != self->ob_descr) { + PyErr_BadArgument(); + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "can only assign array (not \"%.200s\") to array slice", + Py_TYPE(value)->tp_name); + return -1; + } + itemsize = self->ob_descr->itemsize; + /* for 'a[2:1] = ...', the insertion point is 'start', not 'stop' */ + if ((step > 0 && stop < start) || + (step < 0 && stop > start)) + stop = start; + if (step == 1) { + if (slicelength > needed) { + memmove(self->ob_item + (start + needed) * itemsize, + self->ob_item + stop * itemsize, + (Py_SIZE(self) - stop) * itemsize); + if (array_resize(self, Py_SIZE(self) + + needed - slicelength) < 0) + return -1; + } + else if (slicelength < needed) { + if (array_resize(self, Py_SIZE(self) + + needed - slicelength) < 0) + return -1; + memmove(self->ob_item + (start + needed) * itemsize, + self->ob_item + stop * itemsize, + (Py_SIZE(self) - start - needed) * itemsize); + } + if (needed > 0) + memcpy(self->ob_item + start * itemsize, + other->ob_item, needed * itemsize); + return 0; + } + else if (needed == 0) { + /* Delete slice */ + size_t cur; + Py_ssize_t i; + + if (step < 0) { + stop = start + 1; + start = stop + step * (slicelength - 1) - 1; + step = -step; + } + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + Py_ssize_t lim = step - 1; + + if (cur + step >= (size_t)Py_SIZE(self)) + lim = Py_SIZE(self) - cur - 1; + memmove(self->ob_item + (cur - i) * itemsize, + self->ob_item + (cur + 1) * itemsize, + lim * itemsize); + } + cur = start + slicelength * step; + if (cur < (size_t)Py_SIZE(self)) { + memmove(self->ob_item + (cur-slicelength) * itemsize, + self->ob_item + cur * itemsize, + (Py_SIZE(self) - cur) * itemsize); + } + if (array_resize(self, Py_SIZE(self) - slicelength) < 0) + return -1; + return 0; + } + else { + Py_ssize_t cur, i; + + if (needed != slicelength) { + PyErr_Format(PyExc_ValueError, + "attempt to assign array of size %zd " + "to extended slice of size %zd", + needed, slicelength); + return -1; + } + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + memcpy(self->ob_item + cur * itemsize, + other->ob_item + i * itemsize, + itemsize); + } + return 0; + } +} + +static PyMappingMethods array_as_mapping = { + (lenfunc)array_length, + (binaryfunc)array_subscr, + (objobjargproc)array_ass_subscr +}; + +static const void *emptybuf = ""; + +static Py_ssize_t +array_buffer_getreadbuf(arrayobject *self, Py_ssize_t index, const void **ptr) +{ + if ( index != 0 ) { + PyErr_SetString(PyExc_SystemError, + "Accessing non-existent array segment"); + return -1; + } + *ptr = (void *)self->ob_item; + if (*ptr == NULL) + *ptr = emptybuf; + return Py_SIZE(self)*self->ob_descr->itemsize; +} + +static Py_ssize_t +array_buffer_getwritebuf(arrayobject *self, Py_ssize_t index, const void **ptr) +{ + if ( index != 0 ) { + PyErr_SetString(PyExc_SystemError, + "Accessing non-existent array segment"); + return -1; + } + *ptr = (void *)self->ob_item; + if (*ptr == NULL) + *ptr = emptybuf; + return Py_SIZE(self)*self->ob_descr->itemsize; +} + +static Py_ssize_t +array_buffer_getsegcount(arrayobject *self, Py_ssize_t *lenp) +{ + if ( lenp ) + *lenp = Py_SIZE(self)*self->ob_descr->itemsize; + return 1; +} + +static PySequenceMethods array_as_sequence = { + (lenfunc)array_length, /*sq_length*/ + (binaryfunc)array_concat, /*sq_concat*/ + (ssizeargfunc)array_repeat, /*sq_repeat*/ + (ssizeargfunc)array_item, /*sq_item*/ + (ssizessizeargfunc)array_slice, /*sq_slice*/ + (ssizeobjargproc)array_ass_item, /*sq_ass_item*/ + (ssizessizeobjargproc)array_ass_slice, /*sq_ass_slice*/ + (objobjproc)array_contains, /*sq_contains*/ + (binaryfunc)array_inplace_concat, /*sq_inplace_concat*/ + (ssizeargfunc)array_inplace_repeat /*sq_inplace_repeat*/ +}; + +static PyBufferProcs array_as_buffer = { + (readbufferproc)array_buffer_getreadbuf, + (writebufferproc)array_buffer_getwritebuf, + (segcountproc)array_buffer_getsegcount, + NULL, +}; + +static PyObject * +array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + char c; + PyObject *initial = NULL, *it = NULL; + struct arraydescr *descr; + + if (type == &Arraytype && !_PyArg_NoKeywords("array.array()", kwds)) + return NULL; + + if (!PyArg_ParseTuple(args, "c|O:array", &c, &initial)) + return NULL; + + if (!(initial == NULL || PyList_Check(initial) + || PyString_Check(initial) || PyTuple_Check(initial) + || (c == 'u' && PyUnicode_Check(initial)))) { + it = PyObject_GetIter(initial); + if (it == NULL) + return NULL; + /* We set initial to NULL so that the subsequent code + will create an empty array of the appropriate type + and afterwards we can use array_iter_extend to populate + the array. + */ + initial = NULL; + } + for (descr = descriptors; descr->typecode != '\0'; descr++) { + if (descr->typecode == c) { + PyObject *a; + Py_ssize_t len; + + if (initial == NULL || !(PyList_Check(initial) + || PyTuple_Check(initial))) + len = 0; + else + len = PySequence_Size(initial); + + a = newarrayobject(type, len, descr); + if (a == NULL) + return NULL; + + if (len > 0) { + Py_ssize_t i; + for (i = 0; i < len; i++) { + PyObject *v = + PySequence_GetItem(initial, i); + if (v == NULL) { + Py_DECREF(a); + return NULL; + } + if (setarrayitem(a, i, v) != 0) { + Py_DECREF(v); + Py_DECREF(a); + return NULL; + } + Py_DECREF(v); + } + } else if (initial != NULL && PyString_Check(initial)) { + PyObject *t_initial, *v; + t_initial = PyTuple_Pack(1, initial); + if (t_initial == NULL) { + Py_DECREF(a); + return NULL; + } + v = array_fromstring((arrayobject *)a, + t_initial); + Py_DECREF(t_initial); + if (v == NULL) { + Py_DECREF(a); + return NULL; + } + Py_DECREF(v); +#ifdef Py_USING_UNICODE + } else if (initial != NULL && PyUnicode_Check(initial)) { + Py_ssize_t n = PyUnicode_GET_DATA_SIZE(initial); + if (n > 0) { + arrayobject *self = (arrayobject *)a; + char *item = self->ob_item; + item = (char *)PyMem_Realloc(item, n); + if (item == NULL) { + PyErr_NoMemory(); + Py_DECREF(a); + return NULL; + } + self->ob_item = item; + Py_SIZE(self) = n / sizeof(Py_UNICODE); + memcpy(item, PyUnicode_AS_DATA(initial), n); + self->allocated = Py_SIZE(self); + } +#endif + } + if (it != NULL) { + if (array_iter_extend((arrayobject *)a, it) == -1) { + Py_DECREF(it); + Py_DECREF(a); + return NULL; + } + Py_DECREF(it); + } + return a; + } + } + PyErr_SetString(PyExc_ValueError, + "bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)"); + return NULL; +} + + +PyDoc_STRVAR(module_doc, +"This module defines an object type which can efficiently represent\n\ +an array of basic values: characters, integers, floating point\n\ +numbers. Arrays are sequence types and behave very much like lists,\n\ +except that the type of objects stored in them is constrained. The\n\ +type is specified at object creation time by using a type code, which\n\ +is a single character. The following type codes are defined:\n\ +\n\ + Type code C Type Minimum size in bytes \n\ + 'c' character 1 \n\ + 'b' signed integer 1 \n\ + 'B' unsigned integer 1 \n\ + 'u' Unicode character 2 \n\ + 'h' signed integer 2 \n\ + 'H' unsigned integer 2 \n\ + 'i' signed integer 2 \n\ + 'I' unsigned integer 2 \n\ + 'l' signed integer 4 \n\ + 'L' unsigned integer 4 \n\ + 'f' floating point 4 \n\ + 'd' floating point 8 \n\ +\n\ +The constructor is:\n\ +\n\ +array(typecode [, initializer]) -- create a new array\n\ +"); + +PyDoc_STRVAR(arraytype_doc, +"array(typecode [, initializer]) -> array\n\ +\n\ +Return a new array whose items are restricted by typecode, and\n\ +initialized from the optional initializer value, which must be a list,\n\ +string. or iterable over elements of the appropriate type.\n\ +\n\ +Arrays represent basic values and behave very much like lists, except\n\ +the type of objects stored in them is constrained.\n\ +\n\ +Methods:\n\ +\n\ +append() -- append a new item to the end of the array\n\ +buffer_info() -- return information giving the current memory info\n\ +byteswap() -- byteswap all the items of the array\n\ +count() -- return number of occurrences of an object\n\ +extend() -- extend array by appending multiple elements from an iterable\n\ +fromfile() -- read items from a file object\n\ +fromlist() -- append items from the list\n\ +fromstring() -- append items from the string\n\ +index() -- return index of first occurrence of an object\n\ +insert() -- insert a new item into the array at a provided position\n\ +pop() -- remove and return item (default last)\n\ +read() -- DEPRECATED, use fromfile()\n\ +remove() -- remove first occurrence of an object\n\ +reverse() -- reverse the order of the items in the array\n\ +tofile() -- write all items to a file object\n\ +tolist() -- return the array converted to an ordinary list\n\ +tostring() -- return the array converted to a string\n\ +write() -- DEPRECATED, use tofile()\n\ +\n\ +Attributes:\n\ +\n\ +typecode -- the typecode character used to create the array\n\ +itemsize -- the length in bytes of one array item\n\ +"); + +static PyObject *array_iter(arrayobject *ao); + +static PyTypeObject Arraytype = { + PyVarObject_HEAD_INIT(NULL, 0) + "array.array", + sizeof(arrayobject), + 0, + (destructor)array_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)array_repr, /* tp_repr */ + 0, /* tp_as_number*/ + &array_as_sequence, /* tp_as_sequence*/ + &array_as_mapping, /* tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &array_as_buffer, /* tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */ + arraytype_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + array_richcompare, /* tp_richcompare */ + offsetof(arrayobject, weakreflist), /* tp_weaklistoffset */ + (getiterfunc)array_iter, /* tp_iter */ + 0, /* tp_iternext */ + array_methods, /* tp_methods */ + 0, /* tp_members */ + array_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + array_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + +/*********************** Array Iterator **************************/ + +typedef struct { + PyObject_HEAD + Py_ssize_t index; + arrayobject *ao; + PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); +} arrayiterobject; + +static PyTypeObject PyArrayIter_Type; + +#define PyArrayIter_Check(op) PyObject_TypeCheck(op, &PyArrayIter_Type) + +static PyObject * +array_iter(arrayobject *ao) +{ + arrayiterobject *it; + + if (!array_Check(ao)) { + PyErr_BadInternalCall(); + return NULL; + } + + it = PyObject_GC_New(arrayiterobject, &PyArrayIter_Type); + if (it == NULL) + return NULL; + + Py_INCREF(ao); + it->ao = ao; + it->index = 0; + it->getitem = ao->ob_descr->getitem; + PyObject_GC_Track(it); + return (PyObject *)it; +} + +static PyObject * +arrayiter_next(arrayiterobject *it) +{ + assert(PyArrayIter_Check(it)); + if (it->index < Py_SIZE(it->ao)) + return (*it->getitem)(it->ao, it->index++); + return NULL; +} + +static void +arrayiter_dealloc(arrayiterobject *it) +{ + PyObject_GC_UnTrack(it); + Py_XDECREF(it->ao); + PyObject_GC_Del(it); +} + +static int +arrayiter_traverse(arrayiterobject *it, visitproc visit, void *arg) +{ + Py_VISIT(it->ao); + return 0; +} + +static PyTypeObject PyArrayIter_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "arrayiterator", /* tp_name */ + sizeof(arrayiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)arrayiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + 0, /* tp_doc */ + (traverseproc)arrayiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)arrayiter_next, /* tp_iternext */ + 0, /* tp_methods */ +}; + + +/*********************** Install Module **************************/ + +/* No functions in array module. */ +static PyMethodDef a_methods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +PyMODINIT_FUNC +initarray(void) +{ + PyObject *m; + + Arraytype.ob_type = &PyType_Type; + PyArrayIter_Type.ob_type = &PyType_Type; + m = Py_InitModule3("array", a_methods, module_doc); + if (m == NULL) + return; + + Py_INCREF((PyObject *)&Arraytype); + PyModule_AddObject(m, "ArrayType", (PyObject *)&Arraytype); + Py_INCREF((PyObject *)&Arraytype); + PyModule_AddObject(m, "array", (PyObject *)&Arraytype); + /* No need to check the error here, the caller will do that */ +} Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py Sat Apr 10 17:07:54 2010 @@ -0,0 +1,16 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +import py +import sys + +class AppTestArrayModule(AppTestCpythonExtensionBase): + def test_basic(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3]) + assert arr.typecode == 'i' + assert arr.itemsize == 4 + assert arr[2] == 3 + assert len(arr.buffer_info()) == 2 + arr.append(4) + assert arr.tolist() == [1, 2, 3, 4] + assert len(arr) == 4 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sat Apr 10 17:07:54 2010 @@ -552,3 +552,10 @@ w_type2 = from_ref(space, rffi.cast(PyObject, b)) return int(abstract_issubclass_w(space, w_type1, w_type2)) #XXX correct? + at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) +def PyType_GenericAlloc(space, type, nitems): + """This function used an int type for nitems. This might require + changes in your code for properly supporting 64-bit systems.""" + from pypy.module.cpyext.object import _PyObject_NewVar + return _PyObject_NewVar(space, type, nitems) + From jandem at codespeak.net Sat Apr 10 17:11:33 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Sat, 10 Apr 2010 17:11:33 +0200 (CEST) Subject: [pypy-svn] r73629 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410151133.59169282BAD@codespeak.net> Author: jandem Date: Sat Apr 10 17:11:31 2010 New Revision: 73629 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: add slotdef for __getitem__ to make it actually work Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Sat Apr 10 17:11:31 2010 @@ -4,7 +4,8 @@ from pypy.module.cpyext.api import generic_cpy_call, cpython_api, \ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ - ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc + ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc,\ + ssizeargfunc from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt @@ -54,6 +55,13 @@ check_num_args(space, w_args, 0) return generic_cpy_call(space, func_len, w_self) +def wrap_sq_item(space, w_self, w_args, func): + func_target = rffi.cast(ssizeargfunc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + index = space.int_w(space.index(args_w[0])) + return generic_cpy_call(space, func_target, w_self, index) + @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check From xoraxax at codespeak.net Sat Apr 10 17:40:42 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 10 Apr 2010 17:40:42 +0200 (CEST) Subject: [pypy-svn] r73630 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100410154042.9332E282BAD@codespeak.net> Author: xoraxax Date: Sat Apr 10 17:40:41 2010 New Revision: 73630 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Copy header files to _interfaces directory. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sat Apr 10 17:40:41 2010 @@ -1,5 +1,6 @@ import ctypes import sys +import atexit import py @@ -36,11 +37,14 @@ size_t = rffi.ULONG ADDR = lltype.Signed -include_dir = py.path.local(autopath.pypydir) / 'module' / 'cpyext' / 'include' -source_dir = py.path.local(autopath.pypydir) / 'module' / 'cpyext' / 'src' +pypydir = py.path.local(autopath.pypydir) +include_dir = pypydir / 'module' / 'cpyext' / 'include' +source_dir = pypydir / 'module' / 'cpyext' / 'src' +interfaces_dir = pypydir / "_interfaces" include_dirs = [ include_dir, udir, + interfaces_dir, ] class CConfig: @@ -67,6 +71,10 @@ udir.join('pypy_macros.h').write("/* Will be filled later */") globals().update(rffi_platform.configure(CConfig_constants)) +def copy_header_files(): + for name in ("pypy_decl.h", "pypy_macros.h"): + udir.join(name).copy(interfaces_dir / name) + _NOT_SPECIFIED = object() CANNOT_FAIL = object() @@ -557,13 +565,6 @@ for i in range(len(func.argtypes))) body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) functions.append('%s\n%s\n' % (header, body)) - for name, (typ, expr) in GLOBALS.iteritems(): - name_clean = name.replace("#", "") - if not globals_are_pointers: - typ = typ.replace("*", "") - pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) - if not globals_are_pointers and "#" not in name: - pypy_decls.append("#define %s &%s" % (name, name,)) for name in VA_TP_LIST: name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % @@ -571,7 +572,14 @@ pypy_decls.append(header + ';') functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name) export_symbols.append('pypy_va_get_%s' % (name_no_star,)) - + + for name, (typ, expr) in GLOBALS.iteritems(): + name_clean = name.replace("#", "") + if not globals_are_pointers: + typ = typ.replace("*", "") + pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) + if not globals_are_pointers and "#" not in name: + pypy_decls.append("#define %s &%s" % (name, name,)) pypy_decls.append("#endif\n") pypy_decl_h = udir.join('pypy_decl.h') @@ -646,6 +654,7 @@ func.get_wrapper(space).c_name = name setup_init_functions(eci) + copy_header_files() @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): From xoraxax at codespeak.net Sat Apr 10 17:41:26 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 10 Apr 2010 17:41:26 +0200 (CEST) Subject: [pypy-svn] r73631 - pypy/branch/cpython-extension/pypy/_interfaces Message-ID: <20100410154126.C68FE282BAD@codespeak.net> Author: xoraxax Date: Sat Apr 10 17:41:25 2010 New Revision: 73631 Added: pypy/branch/cpython-extension/pypy/_interfaces/ (props changed) Log: Added interfaces directory. From xoraxax at codespeak.net Sat Apr 10 17:45:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 10 Apr 2010 17:45:25 +0200 (CEST) Subject: [pypy-svn] r73632 - pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils Message-ID: <20100410154525.9355B282BAD@codespeak.net> Author: xoraxax Date: Sat Apr 10 17:45:24 2010 New Revision: 73632 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Log: (exarkun, xoraxax) Add cpyext support to distutils. Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Sat Apr 10 17:45:24 2010 @@ -11,6 +11,12 @@ python_build = False +def get_python_inc(plat_specific=0, prefix=None): + from os.path import join as j + if plat_specific: + return j(j(sys.pypy_prefix, "pypy"), "_interfaces") + return j(j(j(j(sys.pypy_prefix, 'pypy'), 'module'), 'cpyext'), 'include') + def get_python_version(): """Return a string containing the major and minor Python version, leaving off the patchlevel. Sample return values could be '1.5' @@ -47,6 +53,7 @@ """Initialize the module as appropriate for POSIX systems.""" g = {} g['EXE'] = "" + g['SO'] = ".so" global _config_vars _config_vars = g From benjamin at codespeak.net Sat Apr 10 22:13:15 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 10 Apr 2010 22:13:15 +0200 (CEST) Subject: [pypy-svn] r73633 - in pypy/trunk/pypy/interpreter/pyparser: . test Message-ID: <20100410201315.170D3282BDC@codespeak.net> Author: benjamin Date: Sat Apr 10 22:13:12 2010 New Revision: 73633 Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py pypy/trunk/pypy/interpreter/pyparser/parser.py pypy/trunk/pypy/interpreter/pyparser/test/test_metaparser.py Log: make dfas a list to optimize access Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/metaparser.py Sat Apr 10 22:13:12 2010 @@ -146,8 +146,8 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - our_id = gram.symbol_ids[name] - gram.dfas[our_id] = (states, self.make_first(gram, name)) + gram.dfas.append((states, self.make_first(gram, name))) + assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parser.py Sat Apr 10 22:13:12 2010 @@ -16,7 +16,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} - self.dfas = {} + self.dfas = [] self.labels = [0] self.token_ids = {} @@ -98,7 +98,7 @@ self.root = None current_node = Node(start, None, [], 0, 0) self.stack = [] - self.stack.append((self.grammar.dfas[start], 0, current_node)) + self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) @@ -124,7 +124,7 @@ state = dfa[0][state_index] return False elif sym_id >= 256: - sub_node_dfa = self.grammar.dfas[sym_id] + sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. if label_index in sub_node_dfa[1]: self.push(sub_node_dfa, next_state, sym_id, lineno, Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_metaparser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_metaparser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_metaparser.py Sat Apr 10 22:13:12 2010 @@ -33,9 +33,8 @@ g = self.gram_for("eval: NAME\n") assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] - assert eval_sym in g.dfas assert g.start == eval_sym - states, first = g.dfas[eval_sym] + states, first = g.dfas[eval_sym - 256] assert states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 @@ -52,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"]][0] + states = g.dfas[g.symbol_ids["foo"] - 256][0] last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] From benjamin at codespeak.net Sun Apr 11 01:57:57 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 01:57:57 +0200 (CEST) Subject: [pypy-svn] r73634 - in pypy/trunk/pypy: rpython/memory/gctransform translator/c Message-ID: <20100410235757.E2E64282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 01:57:55 2010 New Revision: 73634 Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/translator/c/gc.py pypy/trunk/pypy/translator/c/node.py Log: make the gc header inlined by the C compiler This makes us follow strict aliasing correctly as well as simplifying genc a bit. Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Sun Apr 11 01:57:55 2010 @@ -445,11 +445,7 @@ self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) self.malloc_zero_filled = GCClass.malloc_zero_filled - HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR - self._gc_fields = fields = [] - for fldname in HDR._names: - FLDTYPE = getattr(HDR, fldname) - fields.append(('_' + fldname, FLDTYPE)) + HDR = self.HDR = self.gcdata.gc.gcheaderbuilder.HDR size_gc_header = self.gcdata.gc.gcheaderbuilder.size_gc_header vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field) @@ -472,17 +468,13 @@ def finalizer_funcptr_for_type(self, TYPE): return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) - def gc_fields(self): - return self._gc_fields - - def gc_field_values_for(self, obj, needs_hash=False): + def gc_header_for(self, obj, needs_hash=False): hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj) - HDR = self._gc_HDR + HDR = self.HDR withhash, flag = self.gcdata.gc.withhash_flag_is_in_field - result = [] for fldname in HDR._names: - x = getattr(hdr, fldname) if fldname == withhash: + x = getattr(hdr, fldname) TYPE = lltype.typeOf(x) x = lltype.cast_primitive(lltype.Signed, x) if needs_hash: @@ -490,8 +482,8 @@ else: x &= ~flag # clear the flag in the header x = lltype.cast_primitive(TYPE, x) - result.append(x) - return result + setattr(hdr, fldname, x) + return hdr def get_hash_offset(self, T): type_id = self.get_type_id(T) Modified: pypy/trunk/pypy/translator/c/gc.py ============================================================================== --- pypy/trunk/pypy/translator/c/gc.py (original) +++ pypy/trunk/pypy/translator/c/gc.py Sun Apr 11 01:57:55 2010 @@ -20,16 +20,13 @@ def common_gcheader_definition(self, defnode): if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR - return [(name, HDR._flds[name]) for name in HDR._names] - else: - return [] + return defnode.db.gctransformer.HDR + return None def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: raise NotImplementedError - else: - return [] + return None def struct_gcheader_definition(self, defnode): return self.common_gcheader_definition(defnode) @@ -113,11 +110,9 @@ def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: gct = defnode.db.gctransformer - hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) - HDR = gct.HDR - return [getattr(hdr, fldname) for fldname in HDR._names] - else: - return [] + top = top_container(defnode.obj) + return gct.gcheaderbuilder.header_of_object(top)._obj + return None # for structs @@ -196,9 +191,10 @@ def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: - return [lltype.identityhash_nocache(defnode.obj._as_ptr())] - else: - return [] + hdr = lltype.malloc(defnode.db.gctransformer.HDR, immortal=True) + hdr.hash = lltype.identityhash_nocache(defnode.obj._as_ptr()) + return hdr._obj + return None def array_setup(self, arraydefnode): pass @@ -333,13 +329,11 @@ args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) - def common_gcheader_definition(self, defnode): - return defnode.db.gctransformer.gc_fields() - def common_gcheader_initdata(self, defnode): o = top_container(defnode.obj) needs_hash = self.get_prebuilt_hash(o) is not None - return defnode.db.gctransformer.gc_field_values_for(o, needs_hash) + hdr = defnode.db.gctransformer.gc_header_for(o, needs_hash) + return hdr._obj def get_prebuilt_hash(self, obj): # for prebuilt objects that need to have their hash stored and @@ -362,9 +356,14 @@ # all implemented by a single call to a C macro. [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args typename = funcgen.db.gettype(op.result.concretetype) - fieldname = c_vtableinfo.value[2] + tid_field = c_vtableinfo.value[2] + # Fish out the C name of the tid field. + HDR = self.db.gctransformer.HDR + hdr_node = self.db.gettypedefnode(HDR) + fieldname = hdr_node.c_struct_field_name(tid_field) return ( - '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s->_%s, %s);' + '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s->' + '_gcheader.%s, %s);' % (funcgen.expr(op.result), cdecl(typename, ''), funcgen.expr(c_grpptr), Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Sun Apr 11 01:57:55 2010 @@ -88,8 +88,10 @@ if self.varlength != 1: self.normalizedtypename = db.gettype(STRUCT, who_asks=self) if needs_gcheader(self.STRUCT): - for fname, T in db.gcpolicy.struct_gcheader_definition(self): - self.fields.append((fname, db.gettype(T, who_asks=self))) + HDR = db.gcpolicy.struct_gcheader_definition(self) + if HDR is not None: + gc_field = ("_gcheader", db.gettype(HDR, who_asks=self)) + self.fields.append(gc_field) for name in self.fieldnames: T = self.c_struct_field_type(name) if name == STRUCT._arrayfld: @@ -218,8 +220,10 @@ if self.varlength != 1: self.normalizedtypename = db.gettype(ARRAY, who_asks=self) if needs_gcheader(ARRAY): - for fname, T in db.gcpolicy.array_gcheader_definition(self): - self.gcfields.append((fname, db.gettype(T, who_asks=self))) + HDR = db.gcpolicy.array_gcheader_definition(self) + if HDR is not None: + gc_field = ("_gcheader", db.gettype(HDR, who_asks=self)) + self.gcfields.append(gc_field) self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) def computegcinfo(self): @@ -558,12 +562,12 @@ data = [] if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.struct_gcheader_initdata(self)): - data.append(('gcheader%d'%i, thing)) - + gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) + data.append(('gcheader', gc_init)) + for name in defnode.fieldnames: data.append((name, getattr(self.obj, name))) - + # Reasonably, you should only initialise one of the fields of a union # in C. This is possible with the syntax '.fieldname value' or # '.fieldname = value'. But here we don't know which of the @@ -665,12 +669,11 @@ defnode = self.db.gettypedefnode(self.T) yield '{' if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.array_gcheader_initdata(self)): - lines = generic_initializationexpr(self.db, thing, - 'gcheader%d'%i, - '%sgcheader%d' % (decoration, i)) - for line in lines: - yield line + gc_init = self.db.gcpolicy.array_gcheader_initdata(self) + lines = generic_initializationexpr(self.db, gc_init, 'gcheader', + '%sgcheader' % (decoration,)) + for line in lines: + yield line if self.T._hints.get('nolength', False): length = '' else: From benjamin at codespeak.net Sun Apr 11 03:33:08 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 03:33:08 +0200 (CEST) Subject: [pypy-svn] r73635 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100411013308.3BFF2282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 03:33:06 2010 New Revision: 73635 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py Log: rip out unused exception Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Sun Apr 11 03:33:06 2010 @@ -19,9 +19,6 @@ from pypy.rpython.lltypesystem import llmemory import os, sys -class UnhandledRPythonException(Exception): - pass - class CConstant(Symbolic): """ A C-level constant, maybe #define, rendered directly. """ From benjamin at codespeak.net Sun Apr 11 05:11:17 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 05:11:17 +0200 (CEST) Subject: [pypy-svn] r73636 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100411031117.1A83A282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 05:11:15 2010 New Revision: 73636 Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py Log: condense from import Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Sun Apr 11 05:11:15 2010 @@ -1,7 +1,7 @@ import py -from pypy.rlib.rarithmetic import r_int, r_uint, intmask, r_singlefloat -from pypy.rlib.rarithmetic import r_ulonglong, r_longlong, base_int -from pypy.rlib.rarithmetic import normalizedinttype +from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, + r_ulonglong, r_longlong, base_int, + normalizedinttype) from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import Hashable from pypy.tool.tls import tlsobject From benjamin at codespeak.net Sun Apr 11 05:12:47 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 05:12:47 +0200 (CEST) Subject: [pypy-svn] r73637 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20100411031247.EB9D4282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 05:12:46 2010 New Revision: 73637 Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py Log: remove unused log Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Sun Apr 11 05:12:46 2010 @@ -1,4 +1,3 @@ -import py from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, base_int, normalizedinttype) @@ -11,8 +10,6 @@ import struct import weakref -log = py.log.Producer('lltype') - TLS = tlsobject() class _uninitialized(object): Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py Sun Apr 11 05:12:46 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.lib.identity_dict import identity_dict From benjamin at codespeak.net Sun Apr 11 05:19:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 05:19:25 +0200 (CEST) Subject: [pypy-svn] r73638 - pypy/trunk/pypy/rpython/test Message-ID: <20100411031925.05FFD282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 05:19:24 2010 New Revision: 73638 Modified: pypy/trunk/pypy/rpython/test/test_llann.py Log: add import Modified: pypy/trunk/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_llann.py (original) +++ pypy/trunk/pypy/rpython/test/test_llann.py Sun Apr 11 05:19:24 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECTPTR From benjamin at codespeak.net Sun Apr 11 05:22:41 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 05:22:41 +0200 (CEST) Subject: [pypy-svn] r73639 - in pypy/trunk/pypy/translator/tool: . test Message-ID: <20100411032241.C5EB5282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 05:22:40 2010 New Revision: 73639 Modified: pypy/trunk/pypy/translator/tool/cbuild.py pypy/trunk/pypy/translator/tool/test/test_cbuild.py Log: kill unused imports and log Modified: pypy/trunk/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/cbuild.py Sun Apr 11 05:22:40 2010 @@ -1,17 +1,10 @@ -import os, sys, inspect, re, imp - import py from pypy.tool.autopath import pypydir from pypy.translator.platform import host -from pypy.tool.ansi_print import ansi_log from pypy.tool.udir import udir -log = py.log.Producer("cbuild") -py.log.setconsumer("cbuild", ansi_log) - - class ExternalCompilationInfo(object): _ATTRIBUTES = ['pre_include_bits', 'includes', 'include_dirs', Modified: pypy/trunk/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/test/test_cbuild.py Sun Apr 11 05:22:40 2010 @@ -1,6 +1,6 @@ -import py, sys +import py -from pypy.tool.udir import udir +from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo from subprocess import Popen, PIPE, STDOUT From benjamin at codespeak.net Sun Apr 11 05:39:41 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 05:39:41 +0200 (CEST) Subject: [pypy-svn] r73640 - pypy/trunk/pypy/translator/c Message-ID: <20100411033941.CE1B1282BDC@codespeak.net> Author: benjamin Date: Sun Apr 11 05:39:39 2010 New Revision: 73640 Modified: pypy/trunk/pypy/translator/c/database.py Log: reindent Preserving whitespace for the sake of "svn blame" is rather silly. We have "svn blame -x w" for a reason. Modified: pypy/trunk/pypy/translator/c/database.py ============================================================================== --- pypy/trunk/pypy/translator/c/database.py (original) +++ pypy/trunk/pypy/translator/c/database.py Sun Apr 11 05:39:39 2010 @@ -177,58 +177,56 @@ return node def get(self, obj): - # XXX extra indent is preserve svn blame - kind of important IMHO (rxe) - if 1: - if isinstance(obj, CConstant): - return obj.c_name # without further checks - T = typeOf(obj) - if isinstance(T, Primitive) or T == GCREF: - return PrimitiveName[T](obj, self) - elif isinstance(T, Ptr): - if obj: # test if the ptr is non-NULL - try: - container = obj._obj - except lltype.DelayedPointer: - # hack hack hack - name = obj._obj0 - assert name.startswith('delayed!') - n = len('delayed!') - if len(name) == n: - raise - if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): - if obj in self.idelayedfunctionnames: - return self.idelayedfunctionnames[obj][0] - funcname = name[n:] - funcname = self.namespace.uniquename('g_'+funcname) - self.idelayedfunctionnames[obj] = funcname, obj - else: - funcname = None # can't use the name of a - # delayed non-function ptr - self.delayedfunctionptrs.append(obj) - return funcname - # /hack hack hack - else: - # hack hack hack + if isinstance(obj, CConstant): + return obj.c_name # without further checks + T = typeOf(obj) + if isinstance(T, Primitive) or T == GCREF: + return PrimitiveName[T](obj, self) + elif isinstance(T, Ptr): + if obj: # test if the ptr is non-NULL + try: + container = obj._obj + except lltype.DelayedPointer: + # hack hack hack + name = obj._obj0 + assert name.startswith('delayed!') + n = len('delayed!') + if len(name) == n: + raise + if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): if obj in self.idelayedfunctionnames: - # this used to be a delayed function, - # make sure we use the same name - forcename = self.idelayedfunctionnames[obj][0] - node = self.getcontainernode(container, - forcename=forcename) - assert node.ptrname == forcename - return forcename - # /hack hack hack - - if isinstance(container, int): - # special case for tagged odd-valued pointers - return '((%s) %d)' % (cdecl(self.gettype(T), ''), - obj._obj) - node = self.getcontainernode(container) - return node.ptrname + return self.idelayedfunctionnames[obj][0] + funcname = name[n:] + funcname = self.namespace.uniquename('g_'+funcname) + self.idelayedfunctionnames[obj] = funcname, obj + else: + funcname = None # can't use the name of a + # delayed non-function ptr + self.delayedfunctionptrs.append(obj) + return funcname + # /hack hack hack else: - return '((%s) NULL)' % (cdecl(self.gettype(T), ''), ) + # hack hack hack + if obj in self.idelayedfunctionnames: + # this used to be a delayed function, + # make sure we use the same name + forcename = self.idelayedfunctionnames[obj][0] + node = self.getcontainernode(container, + forcename=forcename) + assert node.ptrname == forcename + return forcename + # /hack hack hack + + if isinstance(container, int): + # special case for tagged odd-valued pointers + return '((%s) %d)' % (cdecl(self.gettype(T), ''), + obj._obj) + node = self.getcontainernode(container) + return node.ptrname else: - raise Exception("don't know about %r" % (obj,)) + return '((%s) NULL)' % (cdecl(self.gettype(T), ''), ) + else: + raise Exception("don't know about %r" % (obj,)) def complete(self, show_progress=True): assert not self.completed @@ -304,7 +302,7 @@ self.getcontainernode(value) else: self.get(value) - + while True: while True: while self.pendingsetupnodes: From xoraxax at codespeak.net Sun Apr 11 13:07:35 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 13:07:35 +0200 (CEST) Subject: [pypy-svn] r73641 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411110735.A6D06282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 13:07:33 2010 New Revision: 73641 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Log: Fix cast and test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Sun Apr 11 13:07:33 2010 @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem import lltype, rffi -from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers +from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers, ADDR from pypy.objspace.std.longobject import W_LongObject from pypy.interpreter.error import OperationError @@ -9,7 +9,7 @@ @cpython_api([lltype.Signed], PyObject) def PyLong_FromLong(space, val): """Return a new PyLongObject object from v, or NULL on failure.""" - return space.wrap(val) + return space.newlong(val) @cpython_api([PyObject], rffi.ULONG, error=0) def PyLong_AsUnsignedLong(space, w_long): @@ -25,5 +25,5 @@ can be retrieved from the resulting value using PyLong_AsVoidPtr(). If the integer is larger than LONG_MAX, a positive long integer is returned.""" - return space.wrap(rffi.cast(rffi.LONG, p)) + return space.wrap(rffi.cast(ADDR, p)) From xoraxax at codespeak.net Sun Apr 11 15:52:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 15:52:05 +0200 (CEST) Subject: [pypy-svn] r73642 - in pypy/branch/cpython-extension/pypy/module: cpyext/test imp Message-ID: <20100411135205.DC3D0282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 15:52:04 2010 New Revision: 73642 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py pypy/branch/cpython-extension/pypy/module/imp/importing.py Log: Add support for CPyExt .so/.pyd imports. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Sun Apr 11 15:52:04 2010 @@ -1,4 +1,5 @@ import sys +import os.path import py @@ -122,7 +123,7 @@ cls.space = gettestobjspace(usemodules=['cpyext']) cls.space.getbuiltinmodule("cpyext") - def import_module(self, name, init=None, body=''): + def import_module(self, name, init=None, body='', load_it=True): if init is not None: code = """ #include @@ -149,14 +150,16 @@ kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] mod = compile_module(name, **kwds) - api.load_extension_module(self.space, mod, name) self.name = name - return self.space.getitem( - self.space.sys.get('modules'), - self.space.wrap(name)) + if load_it: + api.load_extension_module(self.space, mod, name) + return self.space.getitem( + self.space.sys.get('modules'), + self.space.wrap(name)) + else: + return os.path.dirname(mod) def import_extension(self, modname, functions, prologue=""): - methods_table = [] codes = [] for funcname, flags, code in functions: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Sun Apr 11 15:52:04 2010 @@ -1,6 +1,5 @@ -import py - from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestImport(BaseApiTest): def test_import(self, space, api): @@ -9,3 +8,12 @@ pdb = api.PyImport_Import(space.wrap("pdb")) assert pdb assert pdb.get("pm") + +class AppTestImportLogic(AppTestCpythonExtensionBase): + def test_import_logic(self): + path = self.import_module(name='foo', load_it=False) + import sys + sys.path.append(path) + import foo + assert foo.fooType + Modified: pypy/branch/cpython-extension/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/imp/importing.py (original) +++ pypy/branch/cpython-extension/pypy/module/imp/importing.py Sun Apr 11 15:52:04 2010 @@ -25,6 +25,11 @@ # PY_CODERESOURCE = 8 IMP_HOOK = 9 +if sys.platform.startswith('win'): + so_extension = ".pyd" +else: + so_extension = ".so" + def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, which is a path without extension. Returns PY_SOURCE, PY_COMPILED or @@ -40,16 +45,18 @@ # look for a lone .pyc file. # The "imp" module does not respect this, and is allowed to find # lone .pyc files. - if not space.config.objspace.lonepycfiles: - return SEARCH_ERROR, None, None - # check the .pyc file - if space.config.objspace.usepycfiles: + if space.config.objspace.usepycfiles and space.config.objspace.lonepycfiles: pycfile = filepart + ".pyc" if os.path.exists(pycfile) and case_ok(pycfile): # existing .pyc file return PY_COMPILED, ".pyc", "rb" + if space.config.objspace.usemodules.cpyext: + pydfile = filepart + so_extension + if os.path.exists(pydfile) and case_ok(pydfile): + return C_EXTENSION, so_extension, "rb" + return SEARCH_ERROR, None, None if sys.platform in ['linux2', 'freebsd']: @@ -332,6 +339,9 @@ except: stream.close() raise + if modtype == C_EXTENSION: + filename = filepart + suffix + return FindInfo(modtype, filename, None, suffix, filemode) except StreamErrors: pass @@ -356,7 +366,7 @@ if find_info.modtype == C_BUILTIN: return space.getbuiltinmodule(find_info.filename, force_init=True) - if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): + if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None if reuse: try: @@ -397,6 +407,12 @@ # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod + elif find_info.modtype == C_EXTENSION and space.config.objspace.usemodules.cpyext: + # the next line is mandantory to init cpyext + space.getbuiltinmodule("cpyext") + from pypy.module.cpyext.api import load_extension_module + load_extension_module(space, find_info.filename, space.str_w(w_modulename)) + return check_sys_modules(space, w_modulename) except OperationError: w_mods = space.sys.get('modules') space.call_method(w_mods, 'pop', w_modulename, space.w_None) From xoraxax at codespeak.net Sun Apr 11 16:34:22 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 16:34:22 +0200 (CEST) Subject: [pypy-svn] r73643 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411143422.66294282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 16:34:20 2010 New Revision: 73643 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Updated TODO file. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Sun Apr 11 16:34:20 2010 @@ -13,9 +13,7 @@ - replace @cpython_api(external=False) by another explicit name: all it does is a lltype function pointer, no C code involved. - - Fix distutil's build_ext to work with cpyext. - - - Extend the import machinery to load .so/.pyd files. + - Fix distutil's build_ext to work with cpyext on windows. - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. @@ -25,4 +23,4 @@ - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which - should be fixed in order to make it nicely work with cpyext. \ No newline at end of file + should be fixed in order to make it nicely work with cpyext. From benjamin at codespeak.net Sun Apr 11 16:37:50 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 16:37:50 +0200 (CEST) Subject: [pypy-svn] r73644 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20100411143750.F2617282B90@codespeak.net> Author: benjamin Date: Sun Apr 11 16:37:49 2010 New Revision: 73644 Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py Log: silence rtyper warning Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parser.py Sun Apr 11 16:37:49 2010 @@ -87,6 +87,7 @@ self.grammar = grammar self.root = None self.stack = None + self.start = 0 def prepare(self, start=-1): """Setup the parser for parsing. From benjamin at codespeak.net Sun Apr 11 18:10:56 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Apr 2010 18:10:56 +0200 (CEST) Subject: [pypy-svn] r73645 - pypy/trunk/pypy/rpython/memory/gctransform Message-ID: <20100411161056.59850282B90@codespeak.net> Author: benjamin Date: Sun Apr 11 18:10:54 2010 New Revision: 73645 Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py Log: simplify setting of prebuilt hash Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Sun Apr 11 18:10:54 2010 @@ -472,17 +472,15 @@ hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj) HDR = self.HDR withhash, flag = self.gcdata.gc.withhash_flag_is_in_field - for fldname in HDR._names: - if fldname == withhash: - x = getattr(hdr, fldname) - TYPE = lltype.typeOf(x) - x = lltype.cast_primitive(lltype.Signed, x) - if needs_hash: - x |= flag # set the flag in the header - else: - x &= ~flag # clear the flag in the header - x = lltype.cast_primitive(TYPE, x) - setattr(hdr, fldname, x) + x = getattr(hdr, withhash) + TYPE = lltype.typeOf(x) + x = lltype.cast_primitive(lltype.Signed, x) + if needs_hash: + x |= flag # set the flag in the header + else: + x &= ~flag # clear the flag in the header + x = lltype.cast_primitive(TYPE, x) + setattr(hdr, withhash, x) return hdr def get_hash_offset(self, T): From xoraxax at codespeak.net Sun Apr 11 19:11:56 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 19:11:56 +0200 (CEST) Subject: [pypy-svn] r73646 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100411171156.58220282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 19:11:54 2010 New Revision: 73646 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import_module.c - copied, changed from r73640, pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Log: Make a dedicated module for test_import_logic, otherwise we poke the broken state of the old module. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Sun Apr 11 19:11:54 2010 @@ -11,9 +11,9 @@ class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): - path = self.import_module(name='foo', load_it=False) + path = self.import_module(name='test_import_module', load_it=False) import sys sys.path.append(path) - import foo - assert foo.fooType + import test_import_module + assert test_import_module.TEST is None Copied: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import_module.c (from r73640, pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c) ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import_module.c Sun Apr 11 19:11:54 2010 @@ -1,287 +1,16 @@ #include "Python.h" -#include "structmember.h" - -typedef struct { - PyObject_HEAD - int foo; /* the context holder */ - PyObject *foo_object; - char *foo_string; - char foo_string_inplace[5]; -} fooobject; - -static PyTypeObject footype; - -static fooobject * -newfooobject(void) -{ - fooobject *foop; - - foop = PyObject_New(fooobject, &footype); - if (foop == NULL) - return NULL; - - foop->foo = 42; - foop->foo_object = NULL; - foop->foo_string = "Hello from PyPy"; - strncpy(foop->foo_string_inplace, "spam", 5); - return foop; -} - - -/* foo methods */ - -static void -foo_dealloc(fooobject *foop) -{ - PyObject_Del(foop); -} - - -/* foo methods-as-attributes */ - -static PyObject * -foo_copy(fooobject *self) -{ - fooobject *foop; - - if ((foop = newfooobject()) == NULL) - return NULL; - - foop->foo = self->foo; - - return (PyObject *)foop; -} - -static PyObject * -foo_unset(fooobject *self) -{ - self->foo_string = NULL; - Py_RETURN_NONE; -} - - -static PyMethodDef foo_methods[] = { - {"copy", (PyCFunction)foo_copy, METH_NOARGS, NULL}, - {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL}, - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -foo_get_name(PyObject *self, void *closure) -{ - return PyString_FromStringAndSize("Foo Example", 11); -} - -static PyObject * -foo_get_foo(PyObject *self, void *closure) -{ - return PyInt_FromLong(((fooobject*)self)->foo); -} - -static PyGetSetDef foo_getseters[] = { - {"name", - (getter)foo_get_name, NULL, - NULL, - NULL}, - {"foo", - (getter)foo_get_foo, NULL, - NULL, - NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -foo_repr(PyObject *self) -{ - PyObject *format; - - format = PyString_FromString(""); - if (format == NULL) return NULL; - return format; -} - -static PyObject * -foo_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - Py_INCREF(kwds); - return kwds; -} - -static PyMemberDef foo_members[] = { - {"int_member", T_INT, offsetof(fooobject, foo), 0, - "A helpful docstring."}, - {"int_member_readonly", T_INT, offsetof(fooobject, foo), READONLY, - "A helpful docstring."}, - {"broken_member", 0xaffe, 0, 0, NULL}, - {"object_member", T_OBJECT, offsetof(fooobject, foo_object), 0, - "A Python object."}, - {"object_member_ex", T_OBJECT_EX, offsetof(fooobject, foo_object), 0, - "A Python object."}, - {"string_member", T_STRING, offsetof(fooobject, foo_string), 0, - "A string."}, - {"string_member_inplace", T_STRING_INPLACE, - offsetof(fooobject, foo_string_inplace), 0, "An inplace string."}, - {"char_member", T_CHAR, offsetof(fooobject, foo_string_inplace), 0, NULL}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject footype = { - PyVarObject_HEAD_INIT(NULL, 0) - "foo.foo", /*tp_name*/ - sizeof(fooobject), /*tp_size*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)foo_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - foo_repr, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - foo_call, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - foo_methods, /*tp_methods*/ - foo_members, /*tp_members*/ - foo_getseters, /*tp_getset*/ -}; - -typedef struct { - PyUnicodeObject HEAD; -} FuuObject; - -static PyObject * -Fuu_escape(PyTypeObject* type, PyObject *args) -{ - Py_RETURN_TRUE; -} - - -static PyMethodDef Fuu_methods[] = { - {"escape", (PyCFunction) Fuu_escape, METH_VARARGS, NULL}, - {NULL} /* Sentinel */ -}; - -PyTypeObject FuuType = { - PyObject_HEAD_INIT(NULL) - 0, - "foo.fuu", - sizeof(FuuObject), - 0, - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES, /*tp_flags*/ - 0, /*tp_doc*/ - - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - - /* Attribute descriptor and subclassing stuff */ - - Fuu_methods,/*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - - 0, /*tp_init*/ - 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - 0, /*tp_new*/ - 0, /*tp_free Low-level free-memory routine */ - 0, /*tp_is_gc For PyObject_IS_GC */ - 0, /*tp_bases*/ - 0, /*tp_mro method resolution order */ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0 /*tp_weaklist*/ -}; - - -/* foo functions */ - -static PyObject * -foo_new(PyObject *self, PyObject *args) -{ - fooobject *foop; - - if ((foop = newfooobject()) == NULL) { - return NULL; - } - - return (PyObject *)foop; -} - - -/* List of functions exported by this module */ - -static PyMethodDef foo_functions[] = { - {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, - {NULL, NULL} /* Sentinel */ -}; - - /* Initialize this module. */ -void initfoo(void) +void inittest_import_module(void) { PyObject *m, *d; - Py_TYPE(&footype) = &PyType_Type; - - /* Workaround for quirk in Visual Studio, see - */ - FuuType.tp_base = &PyUnicode_Type; - - if (PyType_Ready(&footype) < 0) - return; - if (PyType_Ready(&FuuType) < 0) - return; - m = Py_InitModule("foo", foo_functions); + m = Py_InitModule("test_import_module", NULL); if (m == NULL) return; d = PyModule_GetDict(m); if (d) { - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) - return; - PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType); + PyDict_SetItemString(d, "TEST", (PyObject *) Py_None); } /* No need to check the error here, the caller will do that */ } From arigo at codespeak.net Sun Apr 11 19:26:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Apr 2010 19:26:35 +0200 (CEST) Subject: [pypy-svn] r73647 - pypy/trunk/pypy Message-ID: <20100411172635.7DD04282B90@codespeak.net> Author: arigo Date: Sun Apr 11 19:26:33 2010 New Revision: 73647 Modified: pypy/trunk/pypy/conftest.py Log: Unneeded import. Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Sun Apr 11 19:26:33 2010 @@ -1,5 +1,4 @@ import py, sys, os -from py.impl.test.outcome import Failed from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport From agaynor at codespeak.net Sun Apr 11 20:19:24 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 11 Apr 2010 20:19:24 +0200 (CEST) Subject: [pypy-svn] r73650 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100411181924.7243F282B90@codespeak.net> Author: agaynor Date: Sun Apr 11 20:19:22 2010 New Revision: 73650 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Added various macros for the sizes of types. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sun Apr 11 20:19:22 2010 @@ -542,6 +542,18 @@ export_symbols[:] = renamed_symbols else: export_symbols[:] = [sym.replace("#", "") for sym in export_symbols] + + # Generate defines + for macro_name, size in [ + ("SIZEOF_LONG_LONG", rffi.LONGLONG), + ("SIZEOF_VOID_P", rffi.VOIDP), + ("SIZEOF_SIZE_T", rffi.SIZE_T), + ("SIZEOF_LONG", rffi.LONG), + ("SIZEOF_SHORT", rffi.SHORT), + ("SIZEOF_INT", rffi.INT) + ]: + pypy_macros.append("#define %s %s" % (macro_name, rffi.sizeof(size))) + pypy_macros_h = udir.join('pypy_macros.h') pypy_macros_h.write('\n'.join(pypy_macros)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 11 20:19:22 2010 @@ -1,27 +1,6 @@ #ifndef Py_PYTHON_H #define Py_PYTHON_H -// XXX this should be in pyconfig.h - -#define HAVE_LONG_LONG 1 -#define HAVE_STDARG_PROTOTYPES 1 -#define PY_FORMAT_LONG_LONG "ll" -#define PY_LONG_LONG long long -#define SIZEOF_LONG_LONG sizeof(PY_LONG_LONG) -#define PY_FORMAT_SIZE_T "z" -#define SIZEOF_VOID_P sizeof(void *) -#define WITH_DOC_STRINGS -#define HAVE_UNICODE -#define WITHOUT_COMPLEX - -/* PyPy supposes Py_UNICODE == wchar_t */ -#define HAVE_USABLE_WCHAR_T 1 -#ifndef _WIN32 -#define Py_UNICODE_SIZE 4 -#else -#define Py_UNICODE_SIZE 2 -#endif - /* Compat stuff */ #ifndef _WIN32 # include @@ -46,8 +25,6 @@ #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) // obviously wrong -#define SIZEOF_SIZE_T 4 -#define SIZEOF_LONG 4 // from pyport.h #ifdef SIZE_MAX @@ -85,6 +62,8 @@ #include #include +#include "pyconfig.h" + #include "boolobject.h" #include "floatobject.h" #include "methodobject.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h Sun Apr 11 20:19:22 2010 @@ -0,0 +1,30 @@ +#ifndef Py_PYCONFIG_H +#define Py_PYCONFIG_H +#ifdef __cplusplus +extern "C" { +#endif + +#define HAVE_PROTOTYPES 1 +#define STDC_HEADERS 1 + +#define HAVE_LONG_LONG 1 +#define HAVE_STDARG_PROTOTYPES 1 +#define PY_FORMAT_LONG_LONG "ll" +#define PY_LONG_LONG long long +#define PY_FORMAT_SIZE_T "z" +#define WITH_DOC_STRINGS +#define HAVE_UNICODE +#define WITHOUT_COMPLEX + +/* PyPy supposes Py_UNICODE == wchar_t */ +#define HAVE_USABLE_WCHAR_T 1 +#ifndef _WIN32 +#define Py_UNICODE_SIZE 4 +#else +#define Py_UNICODE_SIZE 2 +#endif + +#ifdef __cplusplus +} +#endif +#endif From agaynor at codespeak.net Sun Apr 11 21:51:07 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 11 Apr 2010 21:51:07 +0200 (CEST) Subject: [pypy-svn] r73651 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100411195107.90046282B90@codespeak.net> Author: agaynor Date: Sun Apr 11 21:51:04 2010 New Revision: 73651 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Implemented PyObject_Length Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 11 21:51:04 2010 @@ -96,6 +96,17 @@ def PyObject_Size(space, w_obj): return space.int_w(space.len(w_obj)) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyObject_Length(space, w_obj): + """ + Return the length of object o. If the object o provides either the sequence + and mapping protocols, the sequence length is returned. On error, -1 is + returned. This is the equivalent to the Python expression len(o). + + These functions returned an int type. This might require + changes in your code for properly supporting 64-bit systems.""" + return PyObject_Size(space, w_obj) + @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallable_Check(space, w_obj): """Determine if the object o is callable. Return 1 if the object is callable Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Sun Apr 11 21:51:04 2010 @@ -4568,20 +4568,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyObject_Length(space, o): - """ - - - - Return the length of object o. If the object o provides either the sequence - and mapping protocols, the sequence length is returned. On error, -1 is - returned. This is the equivalent to the Python expression len(o). - - These functions returned an int type. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real) def PyObject_SetItem(space, o, key, v): """Map the object key to the value v. Returns -1 on failure. This is the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sun Apr 11 21:51:04 2010 @@ -64,6 +64,7 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 + assert api.PyObject_Length(space.newlist([space.w_None])) == 1 def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) From xoraxax at codespeak.net Sun Apr 11 22:04:10 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 22:04:10 +0200 (CEST) Subject: [pypy-svn] r73652 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100411200410.23894282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 22:04:09 2010 New Revision: 73652 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Implemented basic GIL handling for CPyExt. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Sun Apr 11 22:04:09 2010 @@ -63,6 +63,7 @@ import pypy.module.cpyext.number import pypy.module.cpyext.sliceobject import pypy.module.cpyext.stubsactive +import pypy.module.cpyext.pystate # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Sun Apr 11 22:04:09 2010 @@ -7,6 +7,7 @@ # include # include # include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_DATA(RTYPE) extern RTYPE #else @@ -41,6 +42,10 @@ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) #endif +#ifndef DL_EXPORT /* declarations for DLL import/export */ +#define DL_EXPORT(RTYPE) RTYPE +#endif + #define statichere static #define Py_MEMCPY memcpy @@ -83,6 +88,7 @@ #include "pycobject.h" #include "bufferobject.h" #include "sliceobject.h" +#include "pystate.h" // XXX This shouldn't be included here #include "structmember.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Sun Apr 11 22:04:09 2010 @@ -0,0 +1,3 @@ +typedef struct _ts { + int initialized; // not used +} PyThreadState; Added: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Sun Apr 11 22:04:09 2010 @@ -0,0 +1,31 @@ +from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ + cpython_struct +from pypy.rpython.lltypesystem import rffi, lltype + + +PyThreadState = lltype.Ptr(cpython_struct("PyThreadState", ())) + + at cpython_api([], PyThreadState, error=CANNOT_FAIL) +def PyEval_SaveThread(space): + """Release the global interpreter lock (if it has been created and thread + support is enabled) and reset the thread state to NULL, returning the + previous thread state (which is not NULL except in PyPy). If the lock has been created, + the current thread must have acquired it. (This function is available even + when thread support is disabled at compile time.)""" + if space.config.objspace.usemodules.thread: + from pypy.module.thread.gil import before_external_call + before_external_call() + return lltype.nullptr(PyThreadState.TO) + + at cpython_api([PyThreadState], lltype.Void) +def PyEval_RestoreThread(space, tstate): + """Acquire the global interpreter lock (if it has been created and thread + support is enabled) and set the thread state to tstate, which must not be + NULL. If the lock has been created, the current thread must not have + acquired it, otherwise deadlock ensues. (This function is available even + when thread support is disabled at compile time.)""" + if space.config.objspace.usemodules.thread: + from pypy.module.thread.gil import after_external_call + after_external_call() + + From agaynor at codespeak.net Sun Apr 11 22:06:23 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 11 Apr 2010 22:06:23 +0200 (CEST) Subject: [pypy-svn] r73653 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100411200623.22384282B90@codespeak.net> Author: agaynor Date: Sun Apr 11 22:06:21 2010 New Revision: 73653 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Switch to a macro for PyObject_Length Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 11 22:06:21 2010 @@ -463,6 +463,8 @@ } \ } while (0) +#define PyObject_Length PyObject_Size + /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 11 22:06:21 2010 @@ -96,17 +96,6 @@ def PyObject_Size(space, w_obj): return space.int_w(space.len(w_obj)) - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyObject_Length(space, w_obj): - """ - Return the length of object o. If the object o provides either the sequence - and mapping protocols, the sequence length is returned. On error, -1 is - returned. This is the equivalent to the Python expression len(o). - - These functions returned an int type. This might require - changes in your code for properly supporting 64-bit systems.""" - return PyObject_Size(space, w_obj) - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallable_Check(space, w_obj): """Determine if the object o is callable. Return 1 if the object is callable Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Sun Apr 11 22:06:21 2010 @@ -64,7 +64,6 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 - assert api.PyObject_Length(space.newlist([space.w_None])) == 1 def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) From xoraxax at codespeak.net Sun Apr 11 22:14:06 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 22:14:06 +0200 (CEST) Subject: [pypy-svn] r73654 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100411201406.A7915282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 22:14:04 2010 New Revision: 73654 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: Add PyCObject_FromVoidPtrAndDesc (untested). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Sun Apr 11 22:14:04 2010 @@ -6,8 +6,9 @@ cpython_struct, PyObjectFields -destructor = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) -PyCObjectStruct = cpython_struct('PyCObject', PyObjectFields + (("destructor", destructor), )) +destructor_short = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) +destructor_long = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real, rffi.VOIDP_real], lltype.Void)) +PyCObjectStruct = cpython_struct('PyCObject', PyObjectFields + (("destructor", destructor_short), )) PyCObject = lltype.Ptr(PyCObjectStruct) @@ -16,23 +17,29 @@ self.space = space class W_PyCObjectFromVoidPtr(W_PyCObject): - def __init__(self, space, voidp): + def __init__(self, space, voidp, desc): W_PyCObject.__init__(self, space) self.voidp = voidp self.pyo = lltype.nullptr(PyCObject.TO) + self.desc = desc def set_pycobject(self, pyo): self.pyo = pyo def __del__(self): if self.pyo and self.pyo.c_destructor: - self.pyo.c_destructor(self.voidp) + if self.desc: + rffi.cast(self.pyo.c_destructor, destructor_long)(self.pyo, self.desc) + else: + self.pyo.c_destructor(self.voidp) - at cpython_api([rffi.VOIDP_real, destructor], PyObject) + + at cpython_api([rffi.VOIDP_real, destructor_short], PyObject) def PyCObject_FromVoidPtr(space, cobj, destr): """Create a PyCObject from the void * cobj. The destr function will be called when the object is reclaimed, unless it is NULL.""" - w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj)) + w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj, + lltype.nullptr(rffi.VOIDP_real.TO))) assert isinstance(w_pycobject, W_PyCObjectFromVoidPtr) pyo = make_ref(space, w_pycobject) pycobject = rffi.cast(PyCObject, pyo) @@ -40,6 +47,20 @@ pycobject.c_destructor = destr return pyo + at cpython_api([rffi.VOIDP_real, rffi.VOIDP_real, destructor_long], PyObject) +def PyCObject_FromVoidPtrAndDesc(space, cobj, desc, destr): + """Create a PyCObject from the void * cobj. The destr + function will be called when the object is reclaimed. The desc argument can + be used to pass extra callback data for the destructor function.""" + w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj, + desc)) + assert isinstance(w_pycobject, W_PyCObjectFromVoidPtr) + pyo = make_ref(space, w_pycobject) + pycobject = rffi.cast(PyCObject, pyo) + w_pycobject.set_pycobject(pycobject) + pycobject.c_destructor = rffi.cast(destructor_short, destr) + return pyo + W_PyCObject.typedef = TypeDef( 'PyCObject', Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Sun Apr 11 22:14:04 2010 @@ -28,4 +28,3 @@ from pypy.module.thread.gil import after_external_call after_external_call() - Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Sun Apr 11 22:14:04 2010 @@ -2,9 +2,9 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.pycobject import destructor +from pypy.module.cpyext.pycobject import destructor_short class TestPyCObject(BaseApiTest): def test_pycobject(self, space, api): - obj = api.PyCObject_FromVoidPtr(rffi.cast(rffi.VOIDP_real, 0), lltype.nullptr(destructor.TO)) + obj = api.PyCObject_FromVoidPtr(rffi.cast(rffi.VOIDP_real, 0), lltype.nullptr(destructor_short.TO)) api.Py_DecRef(obj) From xoraxax at codespeak.net Sun Apr 11 22:22:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 22:22:05 +0200 (CEST) Subject: [pypy-svn] r73655 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411202205.0C176282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 22:22:04 2010 New Revision: 73655 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Log: Add PyObject_Print stub. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Sun Apr 11 22:22:04 2010 @@ -33,3 +33,14 @@ """Return a new PyLongObject object from a C unsigned long, or NULL on failure.""" raise NotImplementedError + +FILE = rffi.VOIDP_real.TO +FILEP = lltype.Ptr(FILE) + at cpython_api([PyObject, FILEP, rffi.INT_real], rffi.INT_real, error=-1) +def PyObject_Print(space, o, fp, flags): + """Print an object o, on file fp. Returns -1 on error. The flags argument + is used to enable certain printing options. The only option currently supported + is Py_PRINT_RAW; if given, the str() of the object is written + instead of the repr().""" + raise NotImplementedError + From xoraxax at codespeak.net Sun Apr 11 22:48:06 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 22:48:06 +0200 (CEST) Subject: [pypy-svn] r73656 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src Message-ID: <20100411204806.4A97C282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 22:48:04 2010 New Revision: 73656 Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Add PyObject_CheckReadBuffer and PyObject_AsReadBuffer. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sun Apr 11 22:48:04 2010 @@ -237,10 +237,10 @@ FUNCTIONS_C = [ # XXX rename to SYMBOLS_C 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', - 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', + 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', - '_PyArg_NoKeywords', + '_PyArg_NoKeywords', 'PyObject_AsReadBuffer', 'PyObject_CheckReadBuffer', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -622,7 +622,8 @@ source_dir / "stringobject.c", source_dir / "mysnprintf.c", source_dir / "pythonrun.c", - source_dir / "bufferobject.c" + source_dir / "bufferobject.c", + source_dir / "object.c", ], separate_module_sources = [code], export_symbols=export_symbols_eci, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 11 22:48:04 2010 @@ -463,9 +463,14 @@ } \ } while (0) -#define PyObject_Length PyObject_Size +/* Copied from CPython ----------------------------- */ +int PyObject_AsReadBuffer(PyObject *, void **, Py_ssize_t *); +int PyObject_CheckReadBuffer(PyObject *); + /* PyPy internal ----------------------------------- */ int PyPyType_Register(PyTypeObject *); +#define PyObject_Length PyObject_Size + #endif Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c Sun Apr 11 22:48:04 2010 @@ -0,0 +1,58 @@ +// contains code from abstract.c +#include + + +static PyObject * +null_error(void) +{ + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_SystemError, + "null argument to internal routine"); + return NULL; +} + +int PyObject_AsReadBuffer(PyObject *obj, + const void **buffer, + Py_ssize_t *buffer_len) +{ + PyBufferProcs *pb; + void *pp; + Py_ssize_t len; + + if (obj == NULL || buffer == NULL || buffer_len == NULL) { + null_error(); + return -1; + } + pb = obj->ob_type->tp_as_buffer; + if (pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL) { + PyErr_SetString(PyExc_TypeError, + "expected a readable buffer object"); + return -1; + } + if ((*pb->bf_getsegcount)(obj, NULL) != 1) { + PyErr_SetString(PyExc_TypeError, + "expected a single-segment buffer object"); + return -1; + } + len = (*pb->bf_getreadbuffer)(obj, 0, &pp); + if (len < 0) + return -1; + *buffer = pp; + *buffer_len = len; + return 0; +} + +int +PyObject_CheckReadBuffer(PyObject *obj) +{ + PyBufferProcs *pb = obj->ob_type->tp_as_buffer; + + if (pb == NULL || + pb->bf_getreadbuffer == NULL || + pb->bf_getsegcount == NULL || + (*pb->bf_getsegcount)(obj, NULL) != 1) + return 0; + return 1; +} From xoraxax at codespeak.net Sun Apr 11 22:56:21 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 22:56:21 +0200 (CEST) Subject: [pypy-svn] r73657 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100411205621.5CCEE282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 22:56:19 2010 New Revision: 73657 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Add const to fix startup. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Sun Apr 11 22:56:19 2010 @@ -464,7 +464,7 @@ } while (0) /* Copied from CPython ----------------------------- */ -int PyObject_AsReadBuffer(PyObject *, void **, Py_ssize_t *); +int PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *); int PyObject_CheckReadBuffer(PyObject *); From xoraxax at codespeak.net Sun Apr 11 23:19:05 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 23:19:05 +0200 (CEST) Subject: [pypy-svn] r73658 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411211905.7484C282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 23:19:03 2010 New Revision: 73658 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Log: Load bridge with RTLD_GLOBAL because we do not use "import libs" on posix with distutils. Fix cast. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sun Apr 11 23:19:03 2010 @@ -495,7 +495,7 @@ # load the bridge, and init structure import ctypes - bridge = ctypes.CDLL(str(modulename)) + bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL) pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI') # populate static data Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Sun Apr 11 23:19:03 2010 @@ -29,7 +29,7 @@ def __del__(self): if self.pyo and self.pyo.c_destructor: if self.desc: - rffi.cast(self.pyo.c_destructor, destructor_long)(self.pyo, self.desc) + rffi.cast(destructor_long, self.pyo.c_destructor)(self.pyo, self.desc) else: self.pyo.c_destructor(self.voidp) From xoraxax at codespeak.net Sun Apr 11 23:21:52 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 23:21:52 +0200 (CEST) Subject: [pypy-svn] r73659 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411212152.E30A8282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 23:21:51 2010 New Revision: 73659 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Log: Fix call. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Sun Apr 11 23:21:51 2010 @@ -29,7 +29,7 @@ def __del__(self): if self.pyo and self.pyo.c_destructor: if self.desc: - rffi.cast(destructor_long, self.pyo.c_destructor)(self.pyo, self.desc) + rffi.cast(destructor_long, self.pyo.c_destructor)(self.voidp, self.desc) else: self.pyo.c_destructor(self.voidp) From xoraxax at codespeak.net Sun Apr 11 23:30:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 11 Apr 2010 23:30:25 +0200 (CEST) Subject: [pypy-svn] r73660 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100411213025.E2323282B90@codespeak.net> Author: xoraxax Date: Sun Apr 11 23:30:24 2010 New Revision: 73660 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Add cast for INT_real comparison. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Sun Apr 11 23:30:24 2010 @@ -141,13 +141,14 @@ return space.repr(w_obj) @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject) -def PyObject_RichCompare(space, w_o1, w_o2, opid): +def PyObject_RichCompare(space, w_o1, w_o2, opid_int): """Compare the values of o1 and o2 using the operation specified by opid, which must be one of Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, or Py_GE, corresponding to <, <=, ==, !=, >, or >= respectively. This is the equivalent of the Python expression o1 op o2, where op is the operator corresponding to opid. Returns the value of the comparison on success, or NULL on failure.""" + opid = rffi.cast(lltype.Signed, opid_int) if opid == Py_LT: return space.lt(w_o1, w_o2) if opid == Py_LE: return space.le(w_o1, w_o2) if opid == Py_EQ: return space.eq(w_o1, w_o2) From afa at codespeak.net Mon Apr 12 01:03:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 01:03:09 +0200 (CEST) Subject: [pypy-svn] r73661 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100411230309.CEFDF282B90@codespeak.net> Author: afa Date: Mon Apr 12 01:03:08 2010 New Revision: 73661 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix test_import on Windows: generate .pyd instead of .dll Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Mon Apr 12 01:03:08 2010 @@ -8,6 +8,8 @@ from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform +from pypy.translator.gensupp import uniquemodulename +from pypy.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException @@ -54,10 +56,17 @@ **kwds ) eci = eci.convert_sources_to_files() + dirname = (udir/uniquemodulename('module')).ensure(dir=1) soname = platform.platform.compile( [], eci, + outputfilename=str(dirname/modname), standalone=False) - return str(soname) + if sys.platform == 'win32': + pydname = soname.new(purebasename=modname, ext='.pyd') + else: + pydname = soname.new(purebasename=modname, ext='.so') + soname.rename(pydname) + return str(pydname) def freeze_refcnts(self): state = self.space.fromcache(State) From benjamin at codespeak.net Mon Apr 12 04:42:07 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 04:42:07 +0200 (CEST) Subject: [pypy-svn] r73662 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20100412024207.AE53A282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 04:42:05 2010 New Revision: 73662 Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py Log: remove pointless line Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parser.py Mon Apr 12 04:42:05 2010 @@ -87,7 +87,6 @@ self.grammar = grammar self.root = None self.stack = None - self.start = 0 def prepare(self, start=-1): """Setup the parser for parsing. From benjamin at codespeak.net Mon Apr 12 04:45:06 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 04:45:06 +0200 (CEST) Subject: [pypy-svn] r73663 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20100412024506.84F38282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 04:45:05 2010 New Revision: 73663 Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py pypy/trunk/pypy/interpreter/pyparser/parser.py Log: add a start attribute to Grammar Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/metaparser.py Mon Apr 12 04:45:05 2010 @@ -128,6 +128,7 @@ def build_grammar(self, grammar_cls): gram = grammar_cls() + gram.start = self.start_symbol names = self.dfas.keys() names.sort() names.remove(self.start_symbol) Modified: pypy/trunk/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/parser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/parser.py Mon Apr 12 04:45:05 2010 @@ -19,6 +19,7 @@ self.dfas = [] self.labels = [0] self.token_ids = {} + self.start = -1 def shared_copy(self): new = self.__class__() From benjamin at codespeak.net Mon Apr 12 04:54:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 04:54:48 +0200 (CEST) Subject: [pypy-svn] r73664 - pypy/trunk/pypy/objspace/std Message-ID: <20100412025448.52A1A282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 04:54:46 2010 New Revision: 73664 Modified: pypy/trunk/pypy/objspace/std/intobject.py pypy/trunk/pypy/objspace/std/smallintobject.py Log: share code between int and smallint Modified: pypy/trunk/pypy/objspace/std/intobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/intobject.py (original) +++ pypy/trunk/pypy/objspace/std/intobject.py Mon Apr 12 04:54:46 2010 @@ -34,6 +34,10 @@ registerimplementation(W_IntObject) +# NB: This code is shared by smallintobject.py, and thus no other Int +# multimethods should be invoked from these implementations. Instead, add an +# alias and then teach copy_multimethods in smallintobject.py to override +# it. See int__Int for example. def int_w__Int(space, w_int1): return int(w_int1.intval) @@ -56,8 +60,7 @@ str__Int = repr__Int -def declare_new_int_comparison(opname, clsname): - # also used by smallintobject.py +def declare_new_int_comparison(opname): import operator from pypy.tool.sourcetools import func_with_new_name op = getattr(operator, opname) @@ -65,18 +68,18 @@ i = w_int1.intval j = w_int2.intval return space.newbool(op(i, j)) - name = "%s__%s_%s" % (opname, clsname, clsname) + name = "%s__Int_Int" % (opname,) return func_with_new_name(f, name), name for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op, "Int") + func, name = declare_new_int_comparison(op) globals()[name] = func def hash__Int(space, w_int1): # unlike CPython, we don't special-case the value -1 in most of our # hash functions, so there is not much sense special-casing it here either. # Make sure this is consistent with the hash of floats and longs. - return int__Int(space, w_int1) + return get_integer(space, w_int1) # coerce def coerce__Int_Int(space, w_int1, w_int2): @@ -217,13 +220,14 @@ raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer negation")) return wrapint(space, x) +get_negint = neg__Int def abs__Int(space, w_int1): if w_int1.intval >= 0: - return pos__Int(space, w_int1) + return get_integer(space, w_int1) else: - return neg__Int(space, w_int1) + return get_negint(space, w_int1) def nonzero__Int(space, w_int1): return space.newbool(w_int1.intval != 0) @@ -240,7 +244,7 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return get_integer(space, w_int1) if b >= LONG_BIT: raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer left shift")) @@ -258,7 +262,7 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return get_integer(space, w_int1) if b >= LONG_BIT: if a < 0: a = -1 @@ -294,10 +298,11 @@ return w_int1 a = w_int1.intval return wrapint(space, a) +get_integer = int__Int pos__Int = int__Int def index__Int(space, w_int1): - return int__Int(space, w_int1) + return get_integer(space, w_int1) def float__Int(space, w_int1): a = w_int1.intval Modified: pypy/trunk/pypy/objspace/std/smallintobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/smallintobject.py (original) +++ pypy/trunk/pypy/objspace/std/smallintobject.py Mon Apr 12 04:54:46 2010 @@ -2,19 +2,18 @@ Implementation of small ints, stored as odd-valued pointers in the translated PyPy. To enable them, see inttype.py. """ +import types from pypy.interpreter.error import OperationError +from pypy.objspace.std import intobject from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all from pypy.objspace.std.multimethod import FailedToImplementArgs from pypy.objspace.std.noneobject import W_NoneObject from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.intobject import W_IntObject, declare_new_int_comparison +from pypy.objspace.std.intobject import W_IntObject, _impl_int_int_pow from pypy.rlib.objectmodel import UnboxedValue -# XXX this is a complete copy of intobject.py. Find a better but still -# XXX annotator-friendly way to share code... - class W_SmallIntObject(W_Object, UnboxedValue): __slots__ = 'intval' @@ -39,215 +38,19 @@ def delegate_SmallInt2Complex(space, w_small): return space.newcomplex(float(w_small.intval), 0.0) +def copy_multimethods(ns): + """Copy integer multimethods for small int.""" + for name, func in intobject.__dict__.iteritems(): + if "__Int" in name: + new_name = name.replace("Int", "SmallInt") + # Copy the function, so the annotator specializes it for + # W_SmallIntObject. + ns[new_name] = types.FunctionType(func.func_code, ns, new_name, + func.func_defaults, + func.func_closure) + ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"] + ns["get_negint"] = ns["neg__SmallInt"] -def int_w__SmallInt(space, w_int1): - return int(w_int1.intval) - -def uint_w__SmallInt(space, w_int1): - intval = w_int1.intval - if intval < 0: - raise OperationError(space.w_ValueError, - space.wrap("cannot convert negative integer to unsigned")) - else: - return r_uint(intval) - -def repr__SmallInt(space, w_int1): - a = w_int1.intval - res = str(a) - return space.wrap(res) - -str__SmallInt = repr__SmallInt - -for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op, "SmallInt") - globals()[name] = func - -def hash__SmallInt(space, w_int1): - # unlike CPython, we don't special-case the value -1 in most of our - # hash functions, so there is not much sense special-casing it here either. - # Make sure this is consistent with the hash of floats and longs. - return int__SmallInt(space, w_int1) - -# coerce -def coerce__SmallInt_SmallInt(space, w_int1, w_int2): - return space.newtuple([w_int1, w_int2]) - - -def add__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - # note that no overflow checking is necessary here: x and y fit into 31 - # bits (or 63 bits respectively), so their sum fits into 32 (or 64) bits. - # wrapint then makes sure that either a tagged int or a normal int is - # created - return wrapint(space, x + y) - -def sub__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - # see comment in add__SmallInt_SmallInt - return wrapint(space, x - y) - -def mul__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - try: - z = ovfcheck(x * y) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer multiplication")) - return wrapint(space, z) - -def div__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer division by zero")) - # no overflow possible - return wrapint(space, x // y) - -floordiv__SmallInt_SmallInt = div__SmallInt_SmallInt - -def truediv__SmallInt_SmallInt(space, w_int1, w_int2): - x = float(w_int1.intval) - y = float(w_int2.intval) - if y == 0.0: - raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division")) - return space.wrap(x / y) - -def mod__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer modulo by zero")) - # no overflow possible - return wrapint(space, x % y) - -def divmod__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer divmod by zero")) - # no overflow possible - z = x // y - m = x % y - return space.newtuple([space.wrap(z), space.wrap(m)]) - -def pow__SmallInt_SmallInt_SmallInt(space, w_int1, w_int2, w_int3): - from pypy.objspace.std.intobject import _impl_int_int_pow - x = w_int1.intval - y = w_int2.intval - z = w_int3.intval - if z == 0: - raise OperationError(space.w_ValueError, - space.wrap("pow() 3rd argument cannot be 0")) - return _impl_int_int_pow(space, x, y, z) - -def pow__SmallInt_SmallInt_None(space, w_int1, w_int2, w_int3): - from pypy.objspace.std.intobject import _impl_int_int_pow - x = w_int1.intval - y = w_int2.intval - return _impl_int_int_pow(space, x, y) - -def neg__SmallInt(space, w_int1): - a = w_int1.intval - # no overflow possible since a fits into 31/63 bits - return wrapint(space, -a) - - -def abs__SmallInt(space, w_int1): - if w_int1.intval >= 0: - return pos__SmallInt(space, w_int1) - else: - return neg__SmallInt(space, w_int1) - -def nonzero__SmallInt(space, w_int1): - return space.newbool(w_int1.intval != 0) - -def invert__SmallInt(space, w_int1): - x = w_int1.intval - a = ~x - return wrapint(space, a) - -def lshift__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return int__SmallInt(space, w_int1) - if b >= LONG_BIT: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - try: - c = ovfcheck_lshift(a, b) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return wrapint(space, c) - -def rshift__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return int__SmallInt(space, w_int1) - if b >= LONG_BIT: - if a < 0: - a = -1 - else: - a = 0 - else: - a = a >> b - return wrapint(space, a) - -def and__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a & b - return wrapint(space, res) - -def xor__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a ^ b - return wrapint(space, res) - -def or__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a | b - return wrapint(space, res) - -# int__SmallInt is supposed to do nothing, unless it has -# a derived integer object, where it should return -# an exact one. -def int__SmallInt(space, w_int1): - if space.is_w(space.type(w_int1), space.w_int): - return w_int1 - a = w_int1.intval - return W_SmallIntObject(a) -pos__SmallInt = int__SmallInt - -def float__SmallInt(space, w_int1): - a = w_int1.intval - x = float(a) - return space.newfloat(x) - -def oct__SmallInt(space, w_int1): - return space.wrap(oct(w_int1.intval)) - -def hex__SmallInt(space, w_int1): - return space.wrap(hex(w_int1.intval)) - -def getnewargs__SmallInt(space, w_int1): - return space.newtuple([wrapint(space, w_int1.intval)]) - +copy_multimethods(globals()) register_all(vars()) From benjamin at codespeak.net Mon Apr 12 04:59:43 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 04:59:43 +0200 (CEST) Subject: [pypy-svn] r73665 - pypy/trunk/pypy/objspace/std Message-ID: <20100412025943.59A42282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 04:59:41 2010 New Revision: 73665 Modified: pypy/trunk/pypy/objspace/std/objspace.py Log: remove old comment about hack Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Mon Apr 12 04:59:41 2010 @@ -249,9 +249,6 @@ raise model.UnwrapError, "cannot unwrap: %r" % w_obj def newint(self, intval): - # this time-critical and circular-imports-funny method was stored - # on 'self' by initialize() - # not sure how bad this is: return wrapint(self, intval) def newfloat(self, floatval): From benjamin at codespeak.net Mon Apr 12 05:10:16 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 05:10:16 +0200 (CEST) Subject: [pypy-svn] r73666 - in pypy/trunk/pypy: interpreter objspace/std Message-ID: <20100412031016.9DD69282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 05:10:15 2010 New Revision: 73666 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/objspace.py Log: rename set_str_keyed_item to setitem_str for consistency with other similar methods Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Apr 12 05:10:15 2010 @@ -42,7 +42,7 @@ def setdictvalue(self, space, attr, w_value, shadows_type=True): w_dict = self.getdict() if w_dict is not None: - space.set_str_keyed_item(w_dict, attr, w_value, shadows_type) + space.setitem_str(w_dict, attr, w_value, shadows_type) return True return False @@ -628,7 +628,7 @@ """shortcut for space.int_w(space.hash(w_obj))""" return self.int_w(self.hash(w_obj)) - def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + def setitem_str(self, w_obj, key, w_value, shadows_type=True): return self.setitem(w_obj, self.wrap(key), w_value) def finditem_str(self, w_obj, key): Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Mon Apr 12 05:10:15 2010 @@ -599,7 +599,7 @@ def STORE_NAME(self, varindex, next_instr): varname = self.getname_u(varindex) w_newvalue = self.popvalue() - self.space.set_str_keyed_item(self.w_locals, varname, w_newvalue) + self.space.setitem_str(self.w_locals, varname, w_newvalue) def DELETE_NAME(self, varindex, next_instr): w_varname = self.getname_w(varindex) @@ -638,7 +638,7 @@ def STORE_GLOBAL(self, nameindex, next_instr): varname = self.getname_u(nameindex) w_newvalue = self.popvalue() - self.space.set_str_keyed_item(self.w_globals, varname, w_newvalue) + self.space.setitem_str(self.w_globals, varname, w_newvalue) def DELETE_GLOBAL(self, nameindex, next_instr): w_varname = self.getname_w(nameindex) Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Mon Apr 12 05:10:15 2010 @@ -102,9 +102,6 @@ else: return None - def set_str_keyed_item(w_dict, key, w_value, shadows_type=True): - w_dict.setitem_str(key, w_value, shadows_type) - # _________________________________________________________________ # implementation methods def impl_getitem(self, w_key): Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Mon Apr 12 05:10:15 2010 @@ -459,11 +459,11 @@ return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) - def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + def setitem_str(self, w_obj, key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): - w_obj.set_str_keyed_item(key, w_value, shadows_type) + w_obj.setitem_str(key, w_value, shadows_type) else: self.setitem(w_obj, self.wrap(key), w_value) From benjamin at codespeak.net Mon Apr 12 05:17:14 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 05:17:14 +0200 (CEST) Subject: [pypy-svn] r73667 - pypy/trunk/pypy/objspace/std Message-ID: <20100412031714.E59EA282B9D@codespeak.net> Author: benjamin Date: Mon Apr 12 05:17:13 2010 New Revision: 73667 Modified: pypy/trunk/pypy/objspace/std/smallintobject.py Log: add rbigint to small int globals Modified: pypy/trunk/pypy/objspace/std/smallintobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/smallintobject.py (original) +++ pypy/trunk/pypy/objspace/std/smallintobject.py Mon Apr 12 05:17:13 2010 @@ -13,6 +13,7 @@ from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.intobject import W_IntObject, _impl_int_int_pow from pypy.rlib.objectmodel import UnboxedValue +from pypy.rlib.rbigint import rbigint class W_SmallIntObject(W_Object, UnboxedValue): From afa at codespeak.net Mon Apr 12 14:03:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 14:03:45 +0200 (CEST) Subject: [pypy-svn] r73668 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412120345.78DCA282B9D@codespeak.net> Author: afa Date: Mon Apr 12 14:03:43 2010 New Revision: 73668 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: Don't blindly wrap the result of a generic_cpy_call: translation fails when the function returns a rffi.INT_real (a "setter", for example) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 12 14:03:43 2010 @@ -756,8 +756,7 @@ state.check_and_raise_exception() return ret - else: - return space.wrap(result) + return result finally: if decref_args: for ref in to_decref: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Mon Apr 12 14:03:43 2010 @@ -53,7 +53,7 @@ def wrap_lenfunc(space, w_self, w_args, func): func_len = rffi.cast(lenfunc, func) check_num_args(space, w_args, 0) - return generic_cpy_call(space, func_len, w_self) + return space.wrap(generic_cpy_call(space, func_len, w_self)) def wrap_sq_item(space, w_self, w_args, func): func_target = rffi.cast(ssizeargfunc, func) From afa at codespeak.net Mon Apr 12 16:38:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 16:38:22 +0200 (CEST) Subject: [pypy-svn] r73671 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412143822.EC839282B9D@codespeak.net> Author: afa Date: Mon Apr 12 16:38:21 2010 New Revision: 73671 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Fix translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Mon Apr 12 16:38:21 2010 @@ -257,9 +257,12 @@ def setter(self, space, w_self, w_value): check_descr(space, w_self, self.pto) - return generic_cpy_call( + res = generic_cpy_call( space, self.getset.c_set, w_self, w_value, self.getset.c_closure) + if rffi.cast(lltype.Signed, res) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() def member_getter(self, space, w_self): check_descr(space, w_self, self.pto) From agaynor at codespeak.net Mon Apr 12 17:00:14 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 12 Apr 2010 17:00:14 +0200 (CEST) Subject: [pypy-svn] r73672 - pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils Message-ID: <20100412150014.89DE0282B9D@codespeak.net> Author: agaynor Date: Mon Apr 12 17:00:13 2010 New Revision: 73672 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Log: os.path.join takes *args Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Mon Apr 12 17:00:13 2010 @@ -14,8 +14,8 @@ def get_python_inc(plat_specific=0, prefix=None): from os.path import join as j if plat_specific: - return j(j(sys.pypy_prefix, "pypy"), "_interfaces") - return j(j(j(j(sys.pypy_prefix, 'pypy'), 'module'), 'cpyext'), 'include') + return j(sys.pypy_prefix, "pypy", "_interfaces") + return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') def get_python_version(): """Return a string containing the major and minor Python version, From afa at codespeak.net Mon Apr 12 17:08:12 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 17:08:12 +0200 (CEST) Subject: [pypy-svn] r73673 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412150812.0FC90282B9D@codespeak.net> Author: afa Date: Mon Apr 12 17:08:11 2010 New Revision: 73673 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Use "PyObject" in header files Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 12 17:08:11 2010 @@ -290,7 +290,7 @@ ("bf_getbuffer", rffi.VOIDP), ("bf_releasebuffer", rffi.VOIDP)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) -cpython_struct('struct _object', PyObjectFields, PyObjectStruct) +cpython_struct('PyObject', PyObjectFields, PyObjectStruct) cpython_struct('PyBufferProcs', PyBufferProcsFields, PyBufferProcs) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) PyVarObject = lltype.Ptr(PyVarObjectStruct) From afa at codespeak.net Mon Apr 12 17:28:37 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 17:28:37 +0200 (CEST) Subject: [pypy-svn] r73674 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412152837.A5543282B9D@codespeak.net> Author: afa Date: Mon Apr 12 17:28:36 2010 New Revision: 73674 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: small beautification of pypy_decl.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 12 17:28:36 2010 @@ -563,11 +563,11 @@ pypy_decls = [] pypy_decls.append("#ifndef PYPY_STANDALONE\n") for name, func in sorted(FUNCTIONS.iteritems()): - restype = db.gettype(func.restype).replace('@', '') + restype = db.gettype(func.restype).replace('@', '').strip() args = [] for i, argtype in enumerate(func.argtypes): arg = db.gettype(argtype) - arg = arg.replace('@', 'arg%d' % (i,)) + arg = arg.replace('@', 'arg%d' % (i,)).strip() args.append(arg) args = ', '.join(args) or "void" header = "%s %s(%s)" % (restype, name, args) From afa at codespeak.net Mon Apr 12 17:41:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Apr 2010 17:41:24 +0200 (CEST) Subject: [pypy-svn] r73675 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412154124.E9202282B9E@codespeak.net> Author: afa Date: Mon Apr 12 17:41:23 2010 New Revision: 73675 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: remove the 'rename' parameter when it does not make sense. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 12 17:41:23 2010 @@ -448,14 +448,14 @@ # Build the bridge DLL, Allow extension DLLs to call # back into Pypy space functions # Do not call this more than once per process -def build_bridge(space, rename=True): +def build_bridge(space): from pypy.module.cpyext.pyobject import make_ref export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() - generate_macros(export_symbols, rename) + generate_macros(export_symbols, rename=True, do_deref=True) # Structure declaration code members = [] @@ -501,8 +501,7 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): name = name.replace("#", "") - if rename: - name = name.replace('Py', 'PyPy') + name = name.replace('Py', 'PyPy') w_obj = eval(expr) ptr = ctypes.c_void_p.in_dll(bridge, name) ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(make_ref(space, w_obj)), @@ -598,12 +597,12 @@ pypy_decl_h.write('\n'.join(pypy_decls)) return functions -def build_eci(build_bridge, export_symbols, code): +def build_eci(building_bridge, export_symbols, code): # Build code and get pointer to the structure kwds = {} export_symbols_eci = export_symbols[:] - if build_bridge: + if building_bridge: if sys.platform == "win32": # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] @@ -632,14 +631,14 @@ return eci -def setup_library(space, rename=False): +def setup_library(space): from pypy.module.cpyext.pyobject import make_ref export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() - generate_macros(export_symbols, rename, False) + generate_macros(export_symbols, rename=False, do_deref=False) functions = generate_decls_and_callbacks(db, [], api_struct=False, globals_are_pointers=False) code = "#include \n" + "\n".join(functions) @@ -652,8 +651,6 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): name = name.replace("#", "") - if rename: - name = name.replace('Py', 'PyPy') w_obj = eval(expr) struct_ptr = make_ref(space, w_obj) struct = rffi.cast(get_structtype_for_ctype(type), struct_ptr)._obj From fijal at codespeak.net Mon Apr 12 21:58:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Apr 2010 21:58:55 +0200 (CEST) Subject: [pypy-svn] r73678 - pypy/branch/decouple-host-opcodes/pypy/translator Message-ID: <20100412195855.B8D8E282B9C@codespeak.net> Author: fijal Date: Mon Apr 12 21:58:54 2010 New Revision: 73678 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py Log: Kill this obscure rebinding Modified: pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/geninterplevel.py Mon Apr 12 21:58:54 2010 @@ -161,11 +161,11 @@ Constant(OperationError).key: late_OperationError, Constant(Arguments).key: late_Arguments, } - u = UniqueList - self.initcode = u() # list of lines for the module's initxxx() - self.latercode = u() # list of generators generating extra lines - # for later in initxxx() -- for recursive - # objects + self.initcode = UniqueList() # list of lines for the module's initxxx() + self.latercode = UniqueList() + # list of generators generating extra lines + # for later in initxxx() -- for recursive + # objects self.namespace = NameManager() self.namespace.make_reserved_names('__doc__ __args__ space goto') self.globaldecl = [] From benjamin at codespeak.net Mon Apr 12 22:27:23 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 22:27:23 +0200 (CEST) Subject: [pypy-svn] r73679 - pypy/trunk/pypy/rpython/test Message-ID: <20100412202723.CA1B1282B9C@codespeak.net> Author: benjamin Date: Mon Apr 12 22:27:21 2010 New Revision: 73679 Modified: pypy/trunk/pypy/rpython/test/test_rlist.py pypy/trunk/pypy/rpython/test/test_rpbc.py Log: add py imports Modified: pypy/trunk/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rlist.py (original) +++ pypy/trunk/pypy/rpython/test/test_rlist.py Mon Apr 12 22:27:21 2010 @@ -1,4 +1,6 @@ import sys +import re +import py from pypy.translator.translator import TranslationContext from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem.lltype import * @@ -11,7 +13,6 @@ from pypy.translator.translator import TranslationContext from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -import re # undo the specialization parameter for n1 in 'get set del'.split(): Modified: pypy/trunk/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rpbc.py (original) +++ pypy/trunk/pypy/rpython/test/test_rpbc.py Mon Apr 12 22:27:21 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.rtyper import RPythonTyper from pypy.rpython.ootypesystem import ootype From fijal at codespeak.net Mon Apr 12 22:42:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Apr 2010 22:42:46 +0200 (CEST) Subject: [pypy-svn] r73680 - in pypy/branch/decouple-host-opcodes/pypy: interpreter tool Message-ID: <20100412204246.66220282B9C@codespeak.net> Author: fijal Date: Mon Apr 12 22:42:44 2010 New Revision: 73680 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/tool/sourcetools.py Log: Check in my fixes. This is still IN-PROGRESS Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Mon Apr 12 22:42:44 2010 @@ -957,7 +957,8 @@ from pypy.interpreter.pycode import PyCode if isinstance(statement, str): compiler = self.createcompiler() - statement = compiler.compile(statement, filename, 'exec', 0) + statement = compiler.compile(statement, filename, 'exec', 0, + hidden_applevel=True) if isinstance(statement, types.CodeType): statement = PyCode._from_code(self, statement, hidden_applevel=hidden_applevel) Modified: pypy/branch/decouple-host-opcodes/pypy/tool/sourcetools.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/tool/sourcetools.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/tool/sourcetools.py Mon Apr 12 22:42:44 2010 @@ -17,7 +17,7 @@ indentation. The shorter triple quotes are choosen automatically. The result is returned as a 1-tuple.""" - if type(func) is not str: + if not isinstance(func, str): doc = func.__doc__ else: doc = func From benjamin at codespeak.net Mon Apr 12 22:45:21 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 22:45:21 +0200 (CEST) Subject: [pypy-svn] r73681 - in pypy/branch/decouple-host-opcodes/pypy/interpreter: . astcompiler pyparser test Message-ID: <20100412204521.75280282B9C@codespeak.net> Author: benjamin Date: Mon Apr 12 22:45:19 2010 New Revision: 73681 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/assemble.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pycompiler.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyparser/pyparse.py pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_compiler.py Log: add a hidden_applevel argument to the compiler Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/astcompiler/assemble.py Mon Apr 12 22:45:19 2010 @@ -400,7 +400,8 @@ self.first_lineno, lnotab, free_names, - cell_names) + cell_names, + self.compile_info.hidden_applevel) def _list_from_dict(d, offset=0): Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pycompiler.py Mon Apr 12 22:45:19 2010 @@ -151,8 +151,9 @@ e.wrap_info(space)) return mod - def compile(self, source, filename, mode, flags): + def compile(self, source, filename, mode, flags, hidden_applevel=False): from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags) + info = CompileInfo(filename, mode, flags, + hidden_applevel=hidden_applevel) mod = self._compile_to_ast(source, info) return self._compile_ast(mod, info) Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyparser/pyparse.py Mon Apr 12 22:45:19 2010 @@ -65,14 +65,18 @@ * encoding: The source encoding. * last_future_import: The line number and offset of the last __future__ import. + * hidden_applevel: Will this code unit and sub units be hidden at the + applevel? """ - def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0)): + def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0), + hidden_applevel=False): self.filename = filename self.mode = mode self.encoding = None self.flags = flags self.last_future_import = future_pos + self.hidden_applevel = hidden_applevel _targets = { Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/test/test_compiler.py Mon Apr 12 22:45:19 2010 @@ -54,6 +54,14 @@ space.raises_w(space.w_SyntaxError, self.compiler.compile_command, ')', '?', mode, 0) + def test_hidden_applevel(self): + code = self.compiler.compile("def f(x): pass", "", "exec", 0, + True) + assert code.hidden_applevel + for w_const in code.co_consts_w: + if isinstance(w_const, PyCode): + assert code.hidden_applevel + def test_indentation_error(self): space = self.space space.raises_w(space.w_SyntaxError, self.compiler.compile_command, From fijal at codespeak.net Mon Apr 12 22:47:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Apr 2010 22:47:55 +0200 (CEST) Subject: [pypy-svn] r73682 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100412204755.C430F282B9C@codespeak.net> Author: fijal Date: Mon Apr 12 22:47:54 2010 New Revision: 73682 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Log: fix Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Mon Apr 12 22:47:54 2010 @@ -958,7 +958,7 @@ if isinstance(statement, str): compiler = self.createcompiler() statement = compiler.compile(statement, filename, 'exec', 0, - hidden_applevel=True) + hidden_applevel=hidden_applevel) if isinstance(statement, types.CodeType): statement = PyCode._from_code(self, statement, hidden_applevel=hidden_applevel) From fijal at codespeak.net Mon Apr 12 22:49:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Apr 2010 22:49:46 +0200 (CEST) Subject: [pypy-svn] r73683 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100412204946.EA0E9282B9C@codespeak.net> Author: fijal Date: Mon Apr 12 22:49:45 2010 New Revision: 73683 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py Log: "typo" Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/gateway.py Mon Apr 12 22:49:45 2010 @@ -827,7 +827,7 @@ def __init__(self, source, filename=None, modname='__builtin__'): # HAAACK (but a good one) if filename is None: - f = sys._getframe(2) + f = sys._getframe(1) filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) filename.__source__ = py.code.Source(source) self.filename = filename From benjamin at codespeak.net Mon Apr 12 22:50:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 12 Apr 2010 22:50:35 +0200 (CEST) Subject: [pypy-svn] r73684 - pypy/trunk/pypy/tool Message-ID: <20100412205035.36DED282B9C@codespeak.net> Author: benjamin Date: Mon Apr 12 22:50:33 2010 New Revision: 73684 Modified: pypy/trunk/pypy/tool/sourcetools.py Log: don't care about python 2.2 Modified: pypy/trunk/pypy/tool/sourcetools.py ============================================================================== --- pypy/trunk/pypy/tool/sourcetools.py (original) +++ pypy/trunk/pypy/tool/sourcetools.py Mon Apr 12 22:50:33 2010 @@ -217,20 +217,15 @@ # ____________________________________________________________ -if sys.version_info >= (2, 3): - def func_with_new_name(func, newname): - """Make a renamed copy of a function.""" - f = new.function(func.func_code, func.func_globals, - newname, func.func_defaults, - func.func_closure) - if func.func_dict: - f.func_dict = {} - f.func_dict.update(func.func_dict) - return f -else: - raise Exception("sorry, Python 2.2 not supported") - # because we need to return a new function object -- impossible in 2.2, - # cannot create functions with closures without using veeeery strange code +def func_with_new_name(func, newname): + """Make a renamed copy of a function.""" + f = new.function(func.func_code, func.func_globals, + newname, func.func_defaults, + func.func_closure) + if func.func_dict: + f.func_dict = {} + f.func_dict.update(func.func_dict) + return f PY_IDENTIFIER = ''.join([(('0' <= chr(i) <= '9' or 'a' <= chr(i) <= 'z' or From fijal at codespeak.net Mon Apr 12 23:28:03 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Apr 2010 23:28:03 +0200 (CEST) Subject: [pypy-svn] r73685 - in pypy/branch/decouple-host-opcodes/pypy: interpreter objspace/flow tool/pytest Message-ID: <20100412212803.4B6EE282B9C@codespeak.net> Author: fijal Date: Mon Apr 12 23:28:01 2010 New Revision: 73685 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py Log: Disallow space.eval(cpycode) and space.exec_(cpycode) alltogether. Move createframe logic to choose based on magic to flow objspace (tentatively, it should be killed alltogether) Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Mon Apr 12 23:28:01 2010 @@ -594,14 +594,7 @@ def createframe(self, code, w_globals, closure=None): "Create an empty PyFrame suitable for this code object." - from pypy.interpreter.pycode import PyCode - assert isinstance(code, PyCode) - magic = code.magic - if magic == self.host_magic: - return self.HostFrameClass(self, code, w_globals, closure) - elif magic == self.our_magic: - return self.FrameClass(self, code, w_globals, closure) - raise ValueError("bad magic %s" % magic) + return self.FrameClass(self, code, w_globals, closure) def allocate_lock(self): """Return an interp-level Lock object if threads are enabled, @@ -940,10 +933,11 @@ import types from pypy.interpreter.pycode import PyCode if isinstance(expression, str): - expression = compile(expression, '?', 'eval') + compiler = self.createcompiler() + expression = compiler.compile(expression, '?', 'eval', 0, + hidden_applevel=hidden_applevel) if isinstance(expression, types.CodeType): - expression = PyCode._from_code(self, expression, - hidden_applevel=hidden_applevel) + raise Exception("space.eval(cpycode) Should not be used") if not isinstance(expression, PyCode): raise TypeError, 'space.eval(): expected a string, code or PyCode object' return expression.exec_code(self, w_globals, w_locals) @@ -960,8 +954,7 @@ statement = compiler.compile(statement, filename, 'exec', 0, hidden_applevel=hidden_applevel) if isinstance(statement, types.CodeType): - statement = PyCode._from_code(self, statement, - hidden_applevel=hidden_applevel) + raise Exception("space.eval(cpycode) Should not be used") if not isinstance(statement, PyCode): raise TypeError, 'space.exec_(): expected a string, code or PyCode object' w_key = self.wrap('__builtins__') Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py Mon Apr 12 23:28:01 2010 @@ -85,6 +85,14 @@ return Constant({}) return self.do_operation('newdict') + def createframe(self, code, w_globals, closure=None): + magic = code.magic + if magic == self.host_magic: + return self.HostFrameClass(self, code, w_globals, closure) + elif magic == self.our_magic: + return self.FrameClass(self, code, w_globals, closure) + raise ValueError("bad magic %s" % magic) + def newtuple(self, args_w): try: content = [self.unwrap(w_arg) for w_arg in args_w] Modified: pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/tool/pytest/appsupport.py Mon Apr 12 23:28:01 2010 @@ -218,7 +218,8 @@ for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) try: - space.exec_(source.compile(), frame.w_globals, w_locals) + space.exec_(str(source), frame.w_globals, w_locals, + filename=__file__) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) From fijal at codespeak.net Tue Apr 13 00:42:09 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 00:42:09 +0200 (CEST) Subject: [pypy-svn] r73686 - pypy/branch/decouple-host-opcodes/pypy/translator/goal Message-ID: <20100412224209.ABE34282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 00:42:08 2010 New Revision: 73686 Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Log: I *think* app_main should not be hidden. Modified: pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/translator/goal/targetpypystandalone.py Tue Apr 13 00:42:08 2010 @@ -234,6 +234,7 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + app.hidden_applevel = False app.can_use_geninterp = False w_dict = app.getwdict(space) entry_point = create_entry_point(space, w_dict) From xoraxax at codespeak.net Tue Apr 13 00:55:11 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 13 Apr 2010 00:55:11 +0200 (CEST) Subject: [pypy-svn] r73687 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100412225511.49C4C282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 13 00:55:09 2010 New Revision: 73687 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: Add new macros from pyport/pyconfig. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Tue Apr 13 00:55:09 2010 @@ -25,7 +25,8 @@ #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1)) #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) -// obviously wrong + +#define Py_USING_UNICODE // from pyport.h #ifdef SIZE_MAX @@ -33,6 +34,32 @@ #else #define PY_SIZE_MAX ((size_t)-1) #endif +/* uintptr_t is the C9X name for an unsigned integral type such that a + * legitimate void* can be cast to uintptr_t and then back to void* again + * without loss of information. Similarly for intptr_t, wrt a signed + * integral type. + */ +#ifdef HAVE_UINTPTR_T +typedef uintptr_t Py_uintptr_t; +typedef intptr_t Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_INT +typedef unsigned int Py_uintptr_t; +typedef int Py_intptr_t; + +#elif SIZEOF_VOID_P <= SIZEOF_LONG +typedef unsigned long Py_uintptr_t; +typedef long Py_intptr_t; + +#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG) +typedef unsigned PY_LONG_LONG Py_uintptr_t; +typedef PY_LONG_LONG Py_intptr_t; + +#else +# error "Python needs a typedef for Py_uintptr_t in pyport.h." +#endif /* HAVE_UINTPTR_T */ + + /* Convert a possibly signed character to a nonnegative int */ /* XXX This assumes characters are 8 bits wide */ From xoraxax at codespeak.net Tue Apr 13 00:56:39 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 13 Apr 2010 00:56:39 +0200 (CEST) Subject: [pypy-svn] r73688 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100412225639.CF6C8282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 13 00:56:38 2010 New Revision: 73688 Added: pypy/branch/cpython-extension/pypy/module/cpyext/numpy.patch Log: Patch against numpy trunk to compile it on PyPy. Added: pypy/branch/cpython-extension/pypy/module/cpyext/numpy.patch ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/numpy.patch Tue Apr 13 00:56:38 2010 @@ -0,0 +1,28 @@ +Index: numpy/distutils/system_info.py +=================================================================== +--- numpy/distutils/system_info.py (revision 8325) ++++ numpy/distutils/system_info.py (working copy) +@@ -201,7 +201,7 @@ + default_x11_include_dirs = ['/usr/X11R6/include','/usr/X11/include', + '/usr/include'] + +-if os.path.join(sys.prefix, 'lib') not in default_lib_dirs: ++if hasattr(sys, "prefix") and os.path.join(sys.prefix, 'lib') not in default_lib_dirs: + default_lib_dirs.insert(0,os.path.join(sys.prefix, 'lib')) + default_include_dirs.append(os.path.join(sys.prefix, 'include')) + default_src_dirs.append(os.path.join(sys.prefix, 'src')) +Index: numpy/distutils/ccompiler.py +=================================================================== +--- numpy/distutils/ccompiler.py (revision 8325) ++++ numpy/distutils/ccompiler.py (working copy) +@@ -19,10 +19,6 @@ + + # hack to set compiler optimizing options. Needs to integrated with something. + import distutils.sysconfig +-_old_init_posix = distutils.sysconfig._init_posix +-def _new_init_posix(): +- _old_init_posix() +- distutils.sysconfig._config_vars['OPT'] = '-Wall -g -O0' + #distutils.sysconfig._init_posix = _new_init_posix + + def replace_method(klass, method_name, func): From fijal at codespeak.net Tue Apr 13 01:32:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 01:32:24 +0200 (CEST) Subject: [pypy-svn] r73689 - pypy/branch/cpython-extension/pypy/translator/c/gcc/test Message-ID: <20100412233224.BFE12282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 01:32:23 2010 New Revision: 73689 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Try to write test for asmgcc & secondary entry points Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Apr 13 01:32:23 2010 @@ -1,10 +1,13 @@ import py -import sys, os +import sys, os, gc from pypy.translator.c.test import test_newgc from pypy.translator.translator import TranslationContext from pypy.translator.c.genc import CStandaloneBuilder from pypy.annotation.listdef import s_list_of_strings from pypy import conftest +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.entrypoint import entrypoint class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved @@ -17,6 +20,8 @@ config = get_pypy_config(translating=True) config.translation.gc = cls.gcpolicy config.translation.gcrootfinder = "asmgcc" + config.translation.secondaryentrypoints = 'x42' + # ^^^ from secondary entry point test return config @classmethod @@ -128,11 +133,6 @@ assert res == 1000 def define_callback_simple(cls): - import gc - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.rpython.annlowlevel import llhelper - from pypy.translator.tool.cbuild import ExternalCompilationInfo - c_source = py.code.Source(""" int mystuff(int(*cb)(int, int)) { @@ -163,6 +163,39 @@ res = self.run('callback_simple') assert res == 4900 + def define_secondary_entrypoint_callback(self): + @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') + def mycallback(a, b): + gc.collect() + return a + b + + c_source = py.code.Source(""" + int mystuff() + { + return callback(40, 2) + callback(3, 4); + } + """) + + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = rffi.llexternal('mystuff', [], lltype.Signed, + compilation_info=eci) + S = lltype.GcStruct('S', ('x', lltype.Signed)) + + def f(): + p = lltype.malloc(S) + p.x = 100 + result = z() + return result * p.x + + def update_config(config): + config.translation.secondaryentrypoints = 'x42' + + f.update_config = update_config + return f + + def test_secondary_entrypoint_callback(self): + res = self.run('secondary_entrypoint_callback') + assert res == 4900 class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): # for the individual tests see From fijal at codespeak.net Tue Apr 13 01:48:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 01:48:01 +0200 (CEST) Subject: [pypy-svn] r73690 - pypy/branch/cpython-extension/pypy/translator/c/gcc/test Message-ID: <20100412234801.BC7A6282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 01:48:00 2010 New Revision: 73690 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Log: fix the test Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Apr 13 01:48:00 2010 @@ -7,7 +7,7 @@ from pypy import conftest from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.entrypoint import entrypoint +from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved @@ -20,8 +20,6 @@ config = get_pypy_config(translating=True) config.translation.gc = cls.gcpolicy config.translation.gcrootfinder = "asmgcc" - config.translation.secondaryentrypoints = 'x42' - # ^^^ from secondary entry point test return config @classmethod @@ -39,11 +37,14 @@ config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() + for f, inputtypes in cls.secondary_entrypoints: + a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() - cbuilder = CStandaloneBuilder(t, main, config=config) + cbuilder = CStandaloneBuilder(t, main, config=config, + secondary_entrypoints=cls.secondary_entrypoints) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) @@ -111,6 +112,7 @@ test_newgc.TestSemiSpaceGC): # for the individual tests see # ====> ../../test/test_newgc.py + secondary_entrypoints = [] def define_large_function(cls): class A(object): @@ -158,7 +160,6 @@ return f - def test_callback_simple(self): res = self.run('callback_simple') assert res == 4900 @@ -170,27 +171,25 @@ return a + b c_source = py.code.Source(""" - int mystuff() + int mystuff2() { return callback(40, 2) + callback(3, 4); } """) eci = ExternalCompilationInfo(separate_module_sources=[c_source]) - z = rffi.llexternal('mystuff', [], lltype.Signed, + z = rffi.llexternal('mystuff2', [], lltype.Signed, compilation_info=eci) S = lltype.GcStruct('S', ('x', lltype.Signed)) + self.secondary_entrypoints.extend(secondary_entrypoints["x42"]) + def f(): p = lltype.malloc(S) p.x = 100 result = z() return result * p.x - def update_config(config): - config.translation.secondaryentrypoints = 'x42' - - f.update_config = update_config return f def test_secondary_entrypoint_callback(self): From fijal at codespeak.net Tue Apr 13 01:55:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 01:55:54 +0200 (CEST) Subject: [pypy-svn] r73691 - pypy/branch/cpython-extension/pypy/translator/c/test Message-ID: <20100412235554.6E095282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 01:55:52 2010 New Revision: 73691 Modified: pypy/branch/cpython-extension/pypy/translator/c/test/test_newgc.py Log: Cut down time when running single test (by -k) Modified: pypy/branch/cpython-extension/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/test/test_newgc.py Tue Apr 13 01:55:52 2010 @@ -66,6 +66,12 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue + keyword = conftest.option.keyword + if keyword: + if keyword.startswith('test_'): + keyword = keyword[len('test_'):] + if keyword not in fullname: + continue prefix, name = fullname.split('_', 1) definefunc = getattr(cls, fullname) func = definefunc.im_func(cls) From fijal at codespeak.net Tue Apr 13 02:03:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:03:14 +0200 (CEST) Subject: [pypy-svn] r73692 - pypy/branch/cpython-extension/pypy/translator/c/gcc/test Message-ID: <20100413000314.91FD8282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:03:13 2010 New Revision: 73692 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Fix the test - now explodes differently Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Apr 13 02:03:13 2010 @@ -8,6 +8,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints +from pypy.rpython.lltypesystem.lloperation import llop class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved @@ -167,6 +168,7 @@ def define_secondary_entrypoint_callback(self): @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') def mycallback(a, b): + llop.gc_stack_bottom(lltype.Void) gc.collect() return a + b From fijal at codespeak.net Tue Apr 13 02:04:41 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:04:41 +0200 (CEST) Subject: [pypy-svn] r73693 - pypy/branch/cpython-extension/pypy/rlib Message-ID: <20100413000441.2B346282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:04:39 2010 New Revision: 73693 Modified: pypy/branch/cpython-extension/pypy/rlib/entrypoint.py Log: Add a comment Modified: pypy/branch/cpython-extension/pypy/rlib/entrypoint.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/entrypoint.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/entrypoint.py Tue Apr 13 02:04:39 2010 @@ -2,6 +2,9 @@ def entrypoint(key, argtypes, c_name=None, relax=False): + """ Note: entrypoint should call llop.gc_stack_bottom on it's own. + That's necessary for making it work with asmgcc and hence JIT + """ def deco(func): secondary_entrypoints.setdefault(key, []).append((func, argtypes)) if c_name is not None: From xoraxax at codespeak.net Tue Apr 13 02:18:17 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 13 Apr 2010 02:18:17 +0200 (CEST) Subject: [pypy-svn] r73694 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413001817.418F7282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 13 02:18:15 2010 New Revision: 73694 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_FromUnicode (untested). Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 13 02:18:15 2010 @@ -147,3 +147,21 @@ if errors: w_errors = rffi.charp2str(encoding) return unicodetype.encode_object(space, w_unicode, w_encoding, w_errors) + + at cpython_api([rffi.CWCHARP, Py_ssize_t], PyObject) +def PyUnicode_FromUnicode(space, wchar_p, length): + """Create a Unicode Object from the Py_UNICODE buffer u of the given size. u + may be NULL which causes the contents to be undefined. It is the user's + responsibility to fill in the needed data. The buffer is copied into the new + object. If the buffer is not NULL, the return value might be a shared object. + Therefore, modification of the resulting Unicode object is only allowed when u + is NULL.""" + if not wchar_p: + raise NotImplementedError + s = rffi.wcharpsize2unicode(wchar_p, length) + ptr = make_ref(space, space.wrap(s)) + return ptr + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def _PyUnicode_AsDefaultEncodedString(space, w_unicode, errors): + return PyUnicode_AsEncodedString(space, w_unicode, lltype.nullptr(rffi.CCHARP.TO), errors) From xoraxax at codespeak.net Tue Apr 13 02:18:30 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 13 Apr 2010 02:18:30 +0200 (CEST) Subject: [pypy-svn] r73695 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100413001830.230F8282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 13 02:18:28 2010 New Revision: 73695 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Log: Define PyMem_Realloc. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pymem.h Tue Apr 13 02:18:28 2010 @@ -13,6 +13,7 @@ #define PyMem_Malloc PyMem_MALLOC #define PyMem_Free PyMem_FREE +#define PyMem_Realloc PyMem_REALLOC /* * Type-oriented memory interface From fijal at codespeak.net Tue Apr 13 02:20:26 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:20:26 +0200 (CEST) Subject: [pypy-svn] r73696 - pypy/branch/cpython-extension/pypy/translator/c/gcc/test Message-ID: <20100413002026.8D187282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:20:25 2010 New Revision: 73696 Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Fix the test Modified: pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Apr 13 02:20:25 2010 @@ -169,7 +169,9 @@ @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') def mycallback(a, b): llop.gc_stack_bottom(lltype.Void) + rffi.stackcounter.stacks_counter += 1 gc.collect() + rffi.stackcounter.stacks_counter -= 1 return a + b c_source = py.code.Source(""" From fijal at codespeak.net Tue Apr 13 02:22:34 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:22:34 +0200 (CEST) Subject: [pypy-svn] r73697 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413002234.9CB9D282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:22:33 2010 New Revision: 73697 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: an attempt to fix asmgcc interaction Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 02:22:33 2010 @@ -353,59 +353,63 @@ # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py + rffi.stackcounter.stacks_counter += 1 boxed_args = () - if not we_are_translated() and DEBUG_WRAPPER: - print >>sys.stderr, callable, - for i, (typ, is_wrapped) in argtypes_enum_ui: - arg = args[i] - if typ is PyObject and is_wrapped: - if arg: - arg_conv = from_ref(space, arg) - else: - arg_conv = None - else: - arg_conv = arg - boxed_args += (arg_conv, ) - state = space.fromcache(State) try: - retval = callable(space, *boxed_args) if not we_are_translated() and DEBUG_WRAPPER: - print >>sys.stderr, " DONE" - except OperationError, e: - failed = True - e.normalize_exception(space) - state.set_exception(e.w_type, e.get_w_value(space)) - except BaseException, e: - failed = True - state.set_exception(space.w_SystemError, space.wrap(str(e))) - if not we_are_translated(): - import traceback - traceback.print_exc() - else: - failed = False - - if failed: - error_value = callable.api_func.error_value - if error_value is CANNOT_FAIL: - raise SystemError("The function '%s' was not supposed to fail" - % (callable.__name__,)) - return error_value - - if callable.api_func.restype is PyObject: - borrowed = callable.api_func.borrowed - if not rffi._isllptr(retval): - retval = make_ref(space, retval, borrowed=borrowed) - if borrowed: - try: - add_borrowed_object(space, retval) - except NullPointerException, e: - if not we_are_translated(): - assert False, "Container not registered by %s" % (callable, ) + print >>sys.stderr, callable, + for i, (typ, is_wrapped) in argtypes_enum_ui: + arg = args[i] + if typ is PyObject and is_wrapped: + if arg: + arg_conv = from_ref(space, arg) else: - raise - elif callable.api_func.restype is not lltype.Void: - retval = rffi.cast(callable.api_func.restype, retval) - return retval + arg_conv = None + else: + arg_conv = arg + boxed_args += (arg_conv, ) + state = space.fromcache(State) + try: + retval = callable(space, *boxed_args) + if not we_are_translated() and DEBUG_WRAPPER: + print >>sys.stderr, " DONE" + except OperationError, e: + failed = True + e.normalize_exception(space) + state.set_exception(e.w_type, e.get_w_value(space)) + except BaseException, e: + failed = True + state.set_exception(space.w_SystemError, space.wrap(str(e))) + if not we_are_translated(): + import traceback + traceback.print_exc() + else: + failed = False + + if failed: + error_value = callable.api_func.error_value + if error_value is CANNOT_FAIL: + raise SystemError("The function '%s' was not supposed to fail" + % (callable.__name__,)) + return error_value + + if callable.api_func.restype is PyObject: + borrowed = callable.api_func.borrowed + if not rffi._isllptr(retval): + retval = make_ref(space, retval, borrowed=borrowed) + if borrowed: + try: + add_borrowed_object(space, retval) + except NullPointerException, e: + if not we_are_translated(): + assert False, "Container not registered by %s" % (callable, ) + else: + raise + elif callable.api_func.restype is not lltype.Void: + retval = rffi.cast(callable.api_func.restype, retval) + return retval + finally: + rffi.stackcounter.stacks_counter -= 1 callable._always_inline_ = True wrapper.__name__ = "wrapper for %r" % (callable, ) return wrapper From fijal at codespeak.net Tue Apr 13 02:31:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:31:23 +0200 (CEST) Subject: [pypy-svn] r73698 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413003123.EF7C5282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:31:22 2010 New Revision: 73698 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: what is not tested is broken. At least don't explode on translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 13 02:31:22 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, build_type_checkers, cpython_api) from pypy.module.cpyext.pyerrors import PyErr_BadArgument -from pypy.module.cpyext.pyobject import PyObject, from_ref +from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref from pypy.module.cpyext.stringobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype From fijal at codespeak.net Tue Apr 13 02:40:04 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 02:40:04 +0200 (CEST) Subject: [pypy-svn] r73699 - pypy/branch/cpython-extension/pypy/rpython/lltypesystem Message-ID: <20100413004004.5C460282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 02:40:02 2010 New Revision: 73699 Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py Log: missing exception_cannot_occur Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py Tue Apr 13 02:40:02 2010 @@ -51,6 +51,7 @@ result = isinstance(s_p, annmodel.SomePtr) return self.bookkeeper.immutablevalue(result) def specialize_call(self, hop): + hop.exception_cannot_occur() return hop.inputconst(lltype.Bool, hop.s_result.const) def llexternal(name, args, result, _callable=None, From fijal at codespeak.net Tue Apr 13 03:44:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 03:44:14 +0200 (CEST) Subject: [pypy-svn] r73700 - pypy/branch/cpython-extension/pypy/translator/c/src Message-ID: <20100413014414.F3FE8282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 03:44:13 2010 New Revision: 73700 Modified: pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c Log: don't define Py_uintptr_t here, unused and conflicts with cpyext Modified: pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/commondefs.h Tue Apr 13 03:44:13 2010 @@ -69,9 +69,6 @@ /********************************************************/ -typedef long Py_intptr_t; -typedef unsigned long Py_uintptr_t; - #if ((-1) >> 1) > 0 # define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) \ ((I) < 0 ? -1-((-1-(I)) >> (J)) : (I) >> (J)) Modified: pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c Tue Apr 13 03:44:13 2010 @@ -225,7 +225,7 @@ #define ulong unsigned long /* assuming >= 32 bits */ #undef uptr -#define uptr Py_uintptr_t +#define uptr unsigned long /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; From fijal at codespeak.net Tue Apr 13 03:48:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 03:48:00 +0200 (CEST) Subject: [pypy-svn] r73701 - pypy/branch/cpython-extension/pypy/translator/platform Message-ID: <20100413014800.1953F282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 03:47:58 2010 New Revision: 73701 Modified: pypy/branch/cpython-extension/pypy/translator/platform/linux.py Log: Let's assume we know what we're doing and decrease the number of warnings Modified: pypy/branch/cpython-extension/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/platform/linux.py (original) +++ pypy/branch/cpython-extension/pypy/translator/platform/linux.py Tue Apr 13 03:47:58 2010 @@ -7,7 +7,7 @@ name = "linux" link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall'] + cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] standalone_only = [] shared_only = ['-fPIC'] so_ext = 'so' From fijal at codespeak.net Tue Apr 13 03:51:26 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 03:51:26 +0200 (CEST) Subject: [pypy-svn] r73702 - pypy/branch/cpython-extension/pypy/translator Message-ID: <20100413015126.66F4F282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 03:51:25 2010 New Revision: 73702 Modified: pypy/branch/cpython-extension/pypy/translator/driver.py Log: fix c tests Modified: pypy/branch/cpython-extension/pypy/translator/driver.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/driver.py (original) +++ pypy/branch/cpython-extension/pypy/translator/driver.py Tue Apr 13 03:51:25 2010 @@ -94,7 +94,7 @@ if setopts is not None: self.config.set(**setopts) - + self.exe_name = exe_name self.extmod_name = extmod_name @@ -525,7 +525,10 @@ def task_compile_c(self): # xxx messy cbuilder = self.cbuilder - cbuilder.compile(exe_name=self.compute_exe_name().basename) + kwds = {} + if self.standalone: + kwds['exe_name'] = self.compute_exe_name().basename + cbuilder.compile(**kwds) if self.standalone: self.c_entryp = cbuilder.executable_name From xoraxax at codespeak.net Tue Apr 13 04:23:41 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 13 Apr 2010 04:23:41 +0200 (CEST) Subject: [pypy-svn] r73704 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413022341.36157282B9C@codespeak.net> Author: xoraxax Date: Tue Apr 13 04:23:39 2010 New Revision: 73704 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Add GIL handling TODO item. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Tue Apr 13 04:23:39 2010 @@ -15,6 +15,8 @@ - Fix distutil's build_ext to work with cpyext on windows. + - Fix GIL handling (e.g. init is called without GIL held). + - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. From fijal at codespeak.net Tue Apr 13 04:29:58 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 04:29:58 +0200 (CEST) Subject: [pypy-svn] r73706 - pypy/branch/cpython-extension/pypy/translator/c/src Message-ID: <20100413022958.CF037282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 04:29:57 2010 New Revision: 73706 Modified: pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c Log: oops, missing cast Modified: pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/obmalloc.c Tue Apr 13 04:29:57 2010 @@ -439,7 +439,7 @@ arenabase = bp; nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * nfreepools == ARENA_SIZE); - excess = (uint) ((Py_uintptr_t)bp & POOL_SIZE_MASK); + excess = (uint) ((uint)bp & POOL_SIZE_MASK); if (excess != 0) { --nfreepools; arenabase += POOL_SIZE - excess; From fijal at codespeak.net Tue Apr 13 05:18:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 05:18:57 +0200 (CEST) Subject: [pypy-svn] r73708 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413031857.33F4A282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 05:18:55 2010 New Revision: 73708 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: try to fight proper handling of stack - catch all exceptions here (there is noone to check for them afterwards anyway) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 05:18:55 2010 @@ -344,6 +344,7 @@ names = callable.api_func.argnames argtypes_enum_ui = unrolling_iterable(enumerate(zip(callable.api_func.argtypes, [name.startswith("w_") for name in names]))) + fatal_value = callable.api_func.restype._defl() @specialize.ll() def wrapper(*args): @@ -354,6 +355,7 @@ # inserted exactly here by the varargs specializer llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py rffi.stackcounter.stacks_counter += 1 + retval = fatal_value boxed_args = () try: if not we_are_translated() and DEBUG_WRAPPER: @@ -391,25 +393,22 @@ if error_value is CANNOT_FAIL: raise SystemError("The function '%s' was not supposed to fail" % (callable.__name__,)) - return error_value + retval = error_value - if callable.api_func.restype is PyObject: + elif callable.api_func.restype is PyObject: borrowed = callable.api_func.borrowed if not rffi._isllptr(retval): retval = make_ref(space, retval, borrowed=borrowed) if borrowed: - try: - add_borrowed_object(space, retval) - except NullPointerException, e: - if not we_are_translated(): - assert False, "Container not registered by %s" % (callable, ) - else: - raise + add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, retval) - return retval - finally: - rffi.stackcounter.stacks_counter -= 1 + except NullPointerException, e: + print "Container not registered by %s" % callable.__name__ + except: + print "Fatal exception encountered" + rffi.stackcounter.stacks_counter -= 1 + return retval callable._always_inline_ = True wrapper.__name__ = "wrapper for %r" % (callable, ) return wrapper From fijal at codespeak.net Tue Apr 13 05:31:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 05:31:16 +0200 (CEST) Subject: [pypy-svn] r73709 - in pypy/branch/cpython-extension/pypy/rlib: . test Message-ID: <20100413033116.5ECE4282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 05:31:14 2010 New Revision: 73709 Modified: pypy/branch/cpython-extension/pypy/rlib/rarithmetic.py pypy/branch/cpython-extension/pypy/rlib/test/test_rarithmetic.py Log: An attempt to fix union of INT_real Modified: pypy/branch/cpython-extension/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/rarithmetic.py Tue Apr 13 05:31:14 2010 @@ -126,6 +126,8 @@ raise OverflowError def compute_restype(self_type, other_type): + if self_type is other_type: + return self_type if other_type in (bool, int, long): if self_type is bool: return int Modified: pypy/branch/cpython-extension/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/test/test_rarithmetic.py Tue Apr 13 05:31:14 2010 @@ -365,3 +365,7 @@ def test_isnan(): assert isnan(NAN) + +def test_int_real_union(): + from pypy.rpython.lltypesystem.rffi import r_int_real + assert compute_restype(r_int_real, r_int_real) is r_int_real From fijal at codespeak.net Tue Apr 13 05:38:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 05:38:35 +0200 (CEST) Subject: [pypy-svn] r73710 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413033835.34250282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 05:38:33 2010 New Revision: 73710 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Try not to mix W_Xxx instance with PyObject pointers Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 05:38:33 2010 @@ -372,7 +372,7 @@ boxed_args += (arg_conv, ) state = space.fromcache(State) try: - retval = callable(space, *boxed_args) + result = callable(space, *boxed_args) if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, " DONE" except OperationError, e: @@ -397,12 +397,14 @@ elif callable.api_func.restype is PyObject: borrowed = callable.api_func.borrowed - if not rffi._isllptr(retval): - retval = make_ref(space, retval, borrowed=borrowed) + if not rffi._isllptr(result): + retval = make_ref(space, result, borrowed=borrowed) + else: + retval = result if borrowed: add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: - retval = rffi.cast(callable.api_func.restype, retval) + retval = rffi.cast(callable.api_func.restype, result) except NullPointerException, e: print "Container not registered by %s" % callable.__name__ except: From fijal at codespeak.net Tue Apr 13 05:47:25 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 05:47:25 +0200 (CEST) Subject: [pypy-svn] r73711 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413034725.847E7282B9C@codespeak.net> Author: fijal Date: Tue Apr 13 05:47:24 2010 New Revision: 73711 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: try differently (an annotator bug workaround) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 05:47:24 2010 @@ -405,9 +405,9 @@ add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) - except NullPointerException, e: + except NullPointerException: print "Container not registered by %s" % callable.__name__ - except: + except Exception: print "Fatal exception encountered" rffi.stackcounter.stacks_counter -= 1 return retval From arigo at codespeak.net Tue Apr 13 11:10:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 11:10:57 +0200 (CEST) Subject: [pypy-svn] r73712 - pypy/build/bot2/pypybuildbot Message-ID: <20100413091057.A74F0282B9D@codespeak.net> Author: arigo Date: Tue Apr 13 11:10:54 2010 New Revision: 73712 Modified: pypy/build/bot2/pypybuildbot/builds.py Log: Compress the nightly builds before sending and storing them. Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Tue Apr 13 11:10:54 2010 @@ -110,8 +110,11 @@ if pypyjit: # upload nightly build, if we're running jit tests - nightly = os.path.expanduser('~/nightly/pypy-c-jit-%(got_revision)s-' + platform) - pypy_c_rel = 'build/pypy/translator/goal/pypy-c' + nightly = os.path.expanduser('~/nightly/pypy-c-jit-%(got_revision)s-' + platform + '.bz2') + self.addStep(ShellCmd( + description="compress pypy-c", + command=["bzip2", "-kf", "pypy/translator/goal/pypy-c"])) + pypy_c_rel = 'build/pypy/translator/goal/pypy-c.bz2' self.addStep(transfer.FileUpload(slavesrc=pypy_c_rel, masterdest=WithProperties(nightly), workdir='.', From arigo at codespeak.net Tue Apr 13 11:35:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 11:35:05 +0200 (CEST) Subject: [pypy-svn] r73713 - pypy/branch/asmgcc-re Message-ID: <20100413093505.829CE282B9D@codespeak.net> Author: arigo Date: Tue Apr 13 11:35:04 2010 New Revision: 73713 Added: pypy/branch/asmgcc-re/ - copied from r73712, pypy/trunk/ Log: A branch in which to try to simplify asmgcc. From afa at codespeak.net Tue Apr 13 12:54:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Apr 2010 12:54:13 +0200 (CEST) Subject: [pypy-svn] r73714 - in pypy/branch/cpython-extension/pypy/module/cpyext: include test Message-ID: <20100413105413.E37A1282B9D@codespeak.net> Author: afa Date: Tue Apr 13 12:54:12 2010 New Revision: 73714 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Log: Add Py_BEGIN_ALLOW_THREADS &co with a minimal test. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Tue Apr 13 12:54:12 2010 @@ -1,3 +1,16 @@ +#ifndef Py_PYSTATE_H +#define Py_PYSTATE_H + typedef struct _ts { int initialized; // not used } PyThreadState; + +#define Py_BEGIN_ALLOW_THREADS { \ + PyThreadState *_save; \ + _save = PyEval_SaveThread(); +#define Py_BLOCK_THREADS PyEval_RestoreThread(_save); +#define Py_UNBLOCK_THREADS _save = PyEval_SaveThread(); +#define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ + } + +#endif /* !Py_PYSTATE_H */ Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py Tue Apr 13 12:54:12 2010 @@ -0,0 +1,19 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestStringObject(AppTestCpythonExtensionBase): + def test_allow_threads(self): + module = self.import_extension('foo', [ + ("test", "METH_NOARGS", + """ + Py_BEGIN_ALLOW_THREADS + { + Py_BLOCK_THREADS + Py_UNBLOCK_THREADS + } + Py_END_ALLOW_THREADS + Py_RETURN_NONE; + """), + ]) + # Should compile at least + module.test() + From afa at codespeak.net Tue Apr 13 12:55:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Apr 2010 12:55:16 +0200 (CEST) Subject: [pypy-svn] r73715 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100413105516.B17A5282B9D@codespeak.net> Author: afa Date: Tue Apr 13 12:55:15 2010 New Revision: 73715 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/sliceobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_bufferobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sliceobject.py (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py (props changed) Log: fixeol From afa at codespeak.net Tue Apr 13 13:09:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Apr 2010 13:09:48 +0200 (CEST) Subject: [pypy-svn] r73716 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100413110948.08FB4282B9D@codespeak.net> Author: afa Date: Tue Apr 13 13:09:47 2010 New Revision: 73716 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Expose more type objects Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 13:09:47 2010 @@ -259,8 +259,12 @@ "Dict": "space.w_dict", "Tuple": "space.w_tuple", "List": "space.w_list", + "String": "space.w_str", "Unicode": "space.w_unicode", - 'Bool': 'space.w_bool', + "Int": "space.w_int", + "Bool": "space.w_bool", + "Float": "space.w_float", + "Long": "space.w_long", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', }.items(): @@ -505,9 +509,11 @@ # populate static data for name, (type, expr) in GLOBALS.iteritems(): + w_obj = eval(expr) name = name.replace("#", "") + INTERPLEVEL_API[name] = w_obj + name = name.replace('Py', 'PyPy') - w_obj = eval(expr) ptr = ctypes.c_void_p.in_dll(bridge, name) ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(make_ref(space, w_obj)), ctypes.c_void_p).value Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Tue Apr 13 13:09:47 2010 @@ -97,5 +97,11 @@ def test_TypeCheck(self, space, api): assert api.PyObject_TypeCheck(space.wrap(1), space.w_int) + assert api.PyObject_TypeCheck(space.wrap(1), api.PyInt_Type) assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_str) + assert api.PyObject_TypeCheck(space.wrap('foo'), api.PyString_Type) assert api.PyObject_TypeCheck(space.wrap('foo'), space.w_object) + assert api.PyObject_TypeCheck(space.wrap(1L), api.PyLong_Type) + assert api.PyObject_TypeCheck(space.wrap(True), api.PyBool_Type) + assert api.PyObject_TypeCheck(space.wrap(1.2), api.PyFloat_Type) + assert api.PyObject_TypeCheck(space.w_int, api.PyType_Type) From afa at codespeak.net Tue Apr 13 13:31:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Apr 2010 13:31:23 +0200 (CEST) Subject: [pypy-svn] r73717 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413113123.28847282B9D@codespeak.net> Author: afa Date: Tue Apr 13 13:31:21 2010 New Revision: 73717 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: unused import Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 13 13:31:21 2010 @@ -17,7 +17,6 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import ObjSpace, unwrap_spec -from pypy.objspace.std.stringobject import W_StringObject from pypy.rlib.entrypoint import entrypoint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import specialize From arigo at codespeak.net Tue Apr 13 13:39:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 13:39:33 +0200 (CEST) Subject: [pypy-svn] r73718 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20100413113933.3D750282B9D@codespeak.net> Author: arigo Date: Tue Apr 13 13:39:31 2010 New Revision: 73718 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, arigo) Make a new test by tweaking the failing test to show a crash at the end (instead of at the beginning). Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Apr 13 13:39:31 2010 @@ -1763,6 +1763,29 @@ """ self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Virtual(node_vtable), otherdescr=Not)', None) + @py.test.mark.xfail + def test_bug_3bis(self): + ops = """ + [p1] + guard_nonnull(p1) [] + guard_class(p1, ConstClass(node_vtable2)) [] + p2 = getfield_gc(p1, descr=nextdescr) + guard_nonnull(12) [] + guard_class(p2, ConstClass(node_vtable)) [] + p3 = getfield_gc(p1, descr=otherdescr) + guard_nonnull(12) [] + guard_class(p3, ConstClass(node_vtable)) [] + p1a = new_with_vtable(ConstClass(node_vtable2)) + p2a = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p3, p2a, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + setfield_gc(p1a, p2a, descr=nextdescr) + setfield_gc(p1a, p3a, descr=otherdescr) + jump(p1a) + """ + self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Virtual(node_vtable), otherdescr=Not)', None) + def test_invalid_loop_1(self): ops = """ [p1] From arigo at codespeak.net Tue Apr 13 14:21:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 14:21:22 +0200 (CEST) Subject: [pypy-svn] r73719 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100413122122.CD838282BF1@codespeak.net> Author: arigo Date: Tue Apr 13 14:21:21 2010 New Revision: 73719 Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Log: (cfbolz, arigo) Clarify the purpose of that strange line. Preparation for the upcoming fix, too. Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Tue Apr 13 14:21:21 2010 @@ -63,13 +63,14 @@ # invariant: if escaped=True, then dependencies is None if not self.escaped: self.escaped = True + self.unique = UNIQUE_NO + # ^^^ always set unique to UNIQUE_NO when we set escaped to True. + # See for example test_find_nodes_store_into_loop_constant_2. if self.dependencies is not None: deps = self.dependencies self.dependencies = None for box in deps: box.mark_escaped() - # see test_find_nodes_store_into_loop_constant_1 for this: - box.unique = UNIQUE_NO def set_unique_nodes(self): if self.fromstart: From arigo at codespeak.net Tue Apr 13 15:30:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 15:30:34 +0200 (CEST) Subject: [pypy-svn] r73720 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20100413133034.CD7EB282B9D@codespeak.net> Author: arigo Date: Tue Apr 13 15:30:32 2010 New Revision: 73720 Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz around, arigo) Fix the two failing tests. The issue is that intersect() did not call mark_escaped() on inputnodes even when they are found to match an escaped exitnode. This was usually not a problem, except in obscure cases shown by the tests. Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Tue Apr 13 15:30:32 2010 @@ -343,12 +343,16 @@ # computed by NodeFinder.find_nodes(). op = loop.operations[-1] assert op.opnum == rop.JUMP - specnodes = [] assert len(self.inputnodes) == len(op.args) - for i in range(len(op.args)): - inputnode = self.inputnodes[i] - exitnode = self.getnode(op.args[i]) - specnodes.append(self.intersect(inputnode, exitnode)) + while True: + self.restart_needed = False + specnodes = [] + for i in range(len(op.args)): + inputnode = self.inputnodes[i] + exitnode = self.getnode(op.args[i]) + specnodes.append(self.intersect(inputnode, exitnode)) + if not self.restart_needed: + break loop.token.specnodes = specnodes def intersect(self, inputnode, exitnode): @@ -363,6 +367,15 @@ return prebuiltNotSpecNode unique = exitnode.unique if unique == UNIQUE_NO: + if inputnode is not self.node_fromstart: + # Mark the input node as escaped, and schedule a complete + # restart of intersect(). This is needed because there is + # an order dependency: calling inputnode.mark_escaped() + # might set the field exitnode.unique to UNIQUE_NO in some + # other node. If inputnode is node_fromstart, there is no + # problem (and it must not be mutated by mark_escaped() then). + inputnode.mark_escaped() + self.restart_needed = True return prebuiltNotSpecNode if unique == UNIQUE_INST: return self.intersect_instance(inputnode, exitnode) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Apr 13 15:30:32 2010 @@ -301,8 +301,8 @@ boxes, getnode = self.find_nodes(ops, 'Virtual(node_vtable, nextdescr=Not)') assert not getnode(boxes.p0).escaped - assert not getnode(boxes.p1).escaped - assert not getnode(boxes.p2).escaped + assert getnode(boxes.p1).escaped + assert getnode(boxes.p2).escaped assert getnode(boxes.p0).fromstart assert getnode(boxes.p1).fromstart assert getnode(boxes.p2).fromstart Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Apr 13 15:30:32 2010 @@ -1740,7 +1740,6 @@ self.optimize_loop(ops, 'Not, VArray(arraydescr2, Not)', expected) - @py.test.mark.xfail def test_bug_3(self): ops = """ [p1] @@ -1761,9 +1760,18 @@ setfield_gc(p1a, p3a, descr=otherdescr) jump(p1a) """ - self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Virtual(node_vtable), otherdescr=Not)', None) + expected = """ + [p2, p3] + guard_class(p2, ConstClass(node_vtable)) [] + guard_class(p3, ConstClass(node_vtable)) [] + setfield_gc(p3, p2, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + p2a = new_with_vtable(ConstClass(node_vtable)) + jump(p2a, p3a) + """ + self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) - @py.test.mark.xfail def test_bug_3bis(self): ops = """ [p1] @@ -1784,7 +1792,17 @@ setfield_gc(p1a, p3a, descr=otherdescr) jump(p1a) """ - self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Virtual(node_vtable), otherdescr=Not)', None) + expected = """ + [p2, p3] + guard_class(p2, ConstClass(node_vtable)) [] + guard_class(p3, ConstClass(node_vtable)) [] + p2a = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p3, p2a, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + jump(p2a, p3a) + """ + self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) def test_invalid_loop_1(self): ops = """ From arigo at codespeak.net Tue Apr 13 15:33:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Apr 2010 15:33:26 +0200 (CEST) Subject: [pypy-svn] r73721 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100413133326.C5C08282B9D@codespeak.net> Author: arigo Date: Tue Apr 13 15:33:25 2010 New Revision: 73721 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: In principle, this was fixed by r73720. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Apr 13 15:33:25 2010 @@ -216,17 +216,8 @@ self._fields[ofs] = fieldvalue def _really_force(self): - if self.source_op is None: - # this case should not occur; I only managed to get it once - # in pypy-c-jit and couldn't reproduce it. The point is - # that it relies on optimizefindnode.py computing exactly - # the right level of specialization, and it seems that there - # is still a corner case where it gets too specialized for - # optimizeopt.py. Let's not crash in release-built - # pypy-c-jit's. XXX find out when - from pypy.rlib.debug import ll_assert - ll_assert(False, "_really_force: source_op is None") - raise InvalidLoop + assert self.source_op is not None + # ^^^ This case should not occur any more (see test_bug_3). # newoperations = self.optimizer.newoperations newoperations.append(self.source_op) From fijal at codespeak.net Tue Apr 13 22:45:06 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 13 Apr 2010 22:45:06 +0200 (CEST) Subject: [pypy-svn] r73722 - in pypy/trunk/pypy/module/_file: . test Message-ID: <20100413204506.7A10F282B9D@codespeak.net> Author: fijal Date: Tue Apr 13 22:45:04 2010 New Revision: 73722 Modified: pypy/trunk/pypy/module/_file/interp_file.py pypy/trunk/pypy/module/_file/interp_stream.py pypy/trunk/pypy/module/_file/test/test_file.py Log: Propagate name to exceptions Modified: pypy/trunk/pypy/module/_file/interp_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_file.py (original) +++ pypy/trunk/pypy/module/_file/interp_file.py Tue Apr 13 22:45:04 2010 @@ -82,6 +82,7 @@ def direct___init__(self, w_name, mode='r', buffering=-1): name = self.space.str_w(w_name) + self.name = name self.direct_close() self.check_mode_ok(mode) stream = streamio.open_file_as_stream(name, mode, buffering) @@ -239,7 +240,7 @@ try: self.direct_fdopen(fd, mode, buffering) except StreamErrors, e: - raise wrap_streamerror(self.space, e) + raise wrap_streamerror(self.space, e, self.name) _exposed_method_names = [] @@ -275,7 +276,7 @@ try: result = self.direct_%(name)s(%(callsig)s) except StreamErrors, e: - raise wrap_streamerror(space, e) + raise wrap_streamerror(space, e, self.name) finally: self.unlock() return %(wrapresult)s Modified: pypy/trunk/pypy/module/_file/interp_stream.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_stream.py (original) +++ pypy/trunk/pypy/module/_file/interp_stream.py Tue Apr 13 22:45:04 2010 @@ -10,16 +10,16 @@ import os -def wrap_streamerror(space, e): +def wrap_streamerror(space, e, filename): if isinstance(e, streamio.StreamError): return OperationError(space.w_ValueError, space.wrap(e.message)) elif isinstance(e, OSError): - return wrap_oserror_as_ioerror(space, e) + return wrap_oserror_as_ioerror(space, e, filename) else: return OperationError(space.w_IOError, space.w_None) -def wrap_oserror_as_ioerror(space, e): +def wrap_oserror_as_ioerror(space, e, filename): assert isinstance(e, OSError) errno = e.errno try: @@ -28,7 +28,8 @@ msg = 'error %d' % errno w_error = space.call_function(space.w_IOError, space.wrap(errno), - space.wrap(msg)) + space.wrap(msg), + space.wrap(filename)) return OperationError(space.w_IOError, w_error) Modified: pypy/trunk/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/test/test_file.py (original) +++ pypy/trunk/pypy/module/_file/test/test_file.py Tue Apr 13 22:45:04 2010 @@ -125,6 +125,13 @@ assert type(res) is str f.close() + def test_oserror_has_filename(self): + try: + f = self.file("file that is clearly not there") + except IOError, e: + assert e.filename == 'file that is clearly not there' + else: + raise Exception("did not raise") def test_readline_mixed_with_read(self): s = '''From MAILER-DAEMON Wed Jan 14 14:42:30 2009 From xoraxax at codespeak.net Wed Apr 14 01:19:09 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 14 Apr 2010 01:19:09 +0200 (CEST) Subject: [pypy-svn] r73723 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413231909.DEE70282B9D@codespeak.net> Author: xoraxax Date: Wed Apr 14 01:19:08 2010 New Revision: 73723 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Insert call_external_function into generic_cpy_call. Translation to be checked. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 14 01:19:08 2010 @@ -715,6 +715,29 @@ unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT + # copied and modified from rffi.py + # The around-handlers are releasing the GIL in a threaded pypy. + # We need tons of care to ensure that no GC operation and no + # exception checking occurs while the GIL is released. + argnames = ', '.join(['a%d' % i for i in range(len(FT.ARGS))]) + source = py.code.Source(""" + def call_external_function(funcptr, %(argnames)s): + # NB. it is essential that no exception checking occurs here! + res = funcptr(%(argnames)s) + return res + """ % locals()) + miniglobals = {'__name__': __name__, # for module name propagation + } + exec source.compile() in miniglobals + call_external_function = miniglobals['call_external_function'] + call_external_function._dont_inline_ = True + call_external_function._annspecialcase_ = 'specialize:ll' + call_external_function._gctransformer_hint_close_stack_ = True + call_external_function = func_with_new_name(call_external_function, + 'ccall_' + name) + # don't inline, as a hack to guarantee that no GC pointer is alive + # anywhere in call_external_function + @specialize.ll() def generic_cpy_call(space, func, *args): boxed_args = () @@ -733,7 +756,7 @@ boxed_args += (arg,) else: boxed_args += (arg,) - result = func(*boxed_args) + result = call_external_function(func, *boxed_args) try: if RESULT_TYPE is PyObject: if result is None: From xoraxax at codespeak.net Wed Apr 14 01:27:17 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 14 Apr 2010 01:27:17 +0200 (CEST) Subject: [pypy-svn] r73724 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100413232717.A9EED282B9D@codespeak.net> Author: xoraxax Date: Wed Apr 14 01:27:15 2010 New Revision: 73724 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Fix comment. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 14 01:27:15 2010 @@ -716,9 +716,8 @@ RESULT_TYPE = FT.RESULT # copied and modified from rffi.py - # The around-handlers are releasing the GIL in a threaded pypy. # We need tons of care to ensure that no GC operation and no - # exception checking occurs while the GIL is released. + # exception checking occurs in call_external_function. argnames = ', '.join(['a%d' % i for i in range(len(FT.ARGS))]) source = py.code.Source(""" def call_external_function(funcptr, %(argnames)s): From fijal at codespeak.net Wed Apr 14 06:18:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 06:18:28 +0200 (CEST) Subject: [pypy-svn] r73727 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100414041828.7803A282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 06:18:26 2010 New Revision: 73727 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Log: Kill the pointless check that killed JIT tests Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Wed Apr 14 06:18:26 2010 @@ -163,8 +163,6 @@ @jit.unroll_safe def dispatch_bytecode(self, co_code, next_instr, ec): space = self.space - # PyFrame is really an abstract class - assert self.__class__ is not pyframe.PyFrame while True: self.last_instr = intmask(next_instr) if not jit.we_are_jitted(): From fijal at codespeak.net Wed Apr 14 07:28:15 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 07:28:15 +0200 (CEST) Subject: [pypy-svn] r73728 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100414052815.CF2D5282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 07:28:14 2010 New Revision: 73728 Added: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (contents, props changed) Modified: pypy/trunk/pypy/jit/tl/tinyframe.py Log: commit today's progress - not very much of that Added: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Wed Apr 14 07:28:14 2010 @@ -0,0 +1,18 @@ + +import py +py.test.skip("work in progress") + +from pypy.jit.tl.tinyframe import * + +class TestCompile(object): + def test_simple(self): + code = compile(''' + LOAD 0 => r1 + LOAD 1 => r0 # comment + # other comment + ADD r0, r1 => r2 + PRINT r2 + ''') + assert disassemble(code) == [ + LOAD, 0, 1, LOAD, 1, 0, ADD, 0, 1, 2, PRINT, 2 + ] Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Wed Apr 14 07:28:14 2010 @@ -18,11 +18,35 @@ function argument always comes in r0 """ +opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD'] +for i, opcode in enumerate(opcodes): + globals()[opcode] = i + class Code(object): _immutable_ = True def __init__(self, code): code = code +class Parser(object): + def compile(self, strrepr): + self.code = [] + for line in strrepr.splitlines(): + comment = line.find('#') + if comment != -1: + line = line[:comment] + line = line.strip() + if not line: + continue + opcode, args = line.split(" ", 1) + getattr(self, 'compile_' + opcode)(args) + + def compile_ADD(self, args): + xxx + + def compile_LOAD(self, args): + self.code.append( + def compile(strrepr): - pass + parser = Parser() + return parser.compile(strrepr) From xoraxax at codespeak.net Wed Apr 14 10:25:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 14 Apr 2010 10:25:25 +0200 (CEST) Subject: [pypy-svn] r73729 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414082525.12833282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 14 10:25:23 2010 New Revision: 73729 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Log: Use generic cpycall also for pycobjects to get the call_external_function_magic. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Wed Apr 14 10:25:23 2010 @@ -29,9 +29,9 @@ def __del__(self): if self.pyo and self.pyo.c_destructor: if self.desc: - rffi.cast(destructor_long, self.pyo.c_destructor)(self.voidp, self.desc) + generic_cpy_call(rffi.cast(destructor_long, self.pyo.c_destructor), self.voidp, self.desc) else: - self.pyo.c_destructor(self.voidp) + generic_cpy_call(self.pyo.c_destructor, self.voidp) @cpython_api([rffi.VOIDP_real, destructor_short], PyObject) From xoraxax at codespeak.net Wed Apr 14 10:28:24 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 14 Apr 2010 10:28:24 +0200 (CEST) Subject: [pypy-svn] r73730 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414082824.BA650282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 14 10:28:23 2010 New Revision: 73730 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Log: Add space argument. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Wed Apr 14 10:28:23 2010 @@ -22,6 +22,7 @@ self.voidp = voidp self.pyo = lltype.nullptr(PyCObject.TO) self.desc = desc + self.space = space def set_pycobject(self, pyo): self.pyo = pyo @@ -29,9 +30,10 @@ def __del__(self): if self.pyo and self.pyo.c_destructor: if self.desc: - generic_cpy_call(rffi.cast(destructor_long, self.pyo.c_destructor), self.voidp, self.desc) + generic_cpy_call(self.space, rffi.cast(destructor_long, + self.pyo.c_destructor), self.voidp, self.desc) else: - generic_cpy_call(self.pyo.c_destructor, self.voidp) + generic_cpy_call(self.space, self.pyo.c_destructor, self.voidp) @cpython_api([rffi.VOIDP_real, destructor_short], PyObject) From xoraxax at codespeak.net Wed Apr 14 11:04:09 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 14 Apr 2010 11:04:09 +0200 (CEST) Subject: [pypy-svn] r73731 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414090409.03567282B9C@codespeak.net> Author: xoraxax Date: Wed Apr 14 11:04:08 2010 New Revision: 73731 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Update TODO file, drop libffi usage in favor of direct fnptr calling, should hold the GIL now when calling. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Wed Apr 14 11:04:08 2010 @@ -20,9 +20,6 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. - - Test cpyext together with the jit. Branch does not work, but it's not - cpyext's fault (it doesn't work even with --withoutmod-cpyext). - - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 14 11:04:08 2010 @@ -676,6 +676,7 @@ setup_init_functions(eci) copy_header_files() +initfunctype = lltype.Ptr(lltype.FuncType([], lltype.Void)) @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): state = space.fromcache(State) @@ -688,14 +689,14 @@ "unable to load extension module '%s': %s", path, e.msg) try: - initfunc = dll.getpointer( - 'init%s' % (name,), [], libffi.ffi_type_void) + initptr = libffi.dlsym(dll.lib, 'init%s' % (name,)) except KeyError: raise operationerrfmt( space.w_ImportError, "function init%s not found in library %s", name, path) - initfunc.call(lltype.Void) + initfunc = rffi.cast(initfunctype, initptr) + generic_cpy_call(space, initfunc) state.check_and_raise_exception() @specialize.ll() From arigo at codespeak.net Wed Apr 14 13:27:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 13:27:53 +0200 (CEST) Subject: [pypy-svn] r73732 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414112753.49650282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 13:27:50 2010 New Revision: 73732 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Perfect elimination order. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 13:27:50 2010 @@ -1,10 +1,59 @@ +""" +the whole algorithm is based on the following paper: +http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.86.1578&rep=rep1&type=pdf + +and the source code of libFIRM (file ir/be/becopyheur.c): +http://pp.info.uni-karlsruhe.de/firm/Main_Page +""" + class DependencyGraph(object): - """ A dependency graph for a given control flow graph (CFG). - Each variable is an node in a graph and we have an edge - if two variables are alive at the same point of time + def __init__(self): + self.neighbours = {} + + def add_node(self, v): + assert v not in self.neighbours, "duplicate vertex %r" % (v,) + self.neighbours[v] = set() + + def add_edge(self, v1, v2): + self.neighbours[v1].add(v2) + self.neighbours[v2].add(v1) + + def perfect_elimination_order(self): + sigma = [set(self.neighbours)] + result = [] + while sigma: + v = sigma[0].pop() + result.append(v) + newsigma = [] + neighb = self.neighbours[v] + for s in sigma: + s1 = set() + s2 = set() + for x in s: + if x in neighb: + s1.add(x) + else: + s2.add(x) + if s1: + newsigma.append(s1) + if s2: + newsigma.append(s2) + sigma = newsigma + result.reverse() + return result + + +_emptyset = frozenset() + + +##class Unit(object): +## """An optimization unit. Represents a phi instruction.""" +## def __init__(self, result, args): +## self.result = result +## self.args = args - the whole algorithm is based on the following paper: - http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.86.1578&rep=rep1&type=pdf - """ +## def optimize(self, depgraph): +## self.queue = [] +## self.insert_qnode(QNode(... Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 13:27:50 2010 @@ -1,16 +1,25 @@ - -from pypy.translator.translator import graphof -from pypy.annotation.annrpython import RPythonAnnotator from pypy.tool.algo.color import DependencyGraph -def test_one(): - def f(a, b, c): - d = a + b - e = b + c - f = d + e - return d + e + f - a = RPythonAnnotator() - a.build_types(f, [int, int, int]) - graph = graphof(a.translator, f) - dep_graph = DependencyGraph(graph) +def test_perfect_elimination_order(): + dg = DependencyGraph() + dg.add_node('a') + dg.add_node('b') + dg.add_node('c') + dg.add_node('d') + dg.add_node('e') + dg.add_edge('a', 'b') + dg.add_edge('a', 'd') + dg.add_edge('d', 'b') + dg.add_edge('d', 'e') + dg.add_edge('b', 'c') + dg.add_edge('b', 'e') + dg.add_edge('e', 'c') + order = list(dg.perfect_elimination_order()) + assert len(order) == 5 + assert ''.join(order) in [ + 'adbce', 'adbec', 'adcbe', 'adceb', 'adebc', 'adecb', + 'acbde', 'acbed', 'acdbe', 'acdeb', 'acebd', 'acedb', + 'cebad', 'cebda', 'ceabd', 'ceadb', 'cedba', 'cedab', + 'cabde', 'cabed', 'cadbe', 'cadeb', 'caebd', 'caedb', + ] From arigo at codespeak.net Wed Apr 14 13:30:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 13:30:41 +0200 (CEST) Subject: [pypy-svn] r73733 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414113041.30CF6282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 13:30:39 2010 New Revision: 73733 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Rename it to Lexicographic breath-first ordering. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 13:30:39 2010 @@ -20,12 +20,12 @@ self.neighbours[v1].add(v2) self.neighbours[v2].add(v1) - def perfect_elimination_order(self): + def lexicographic_order(self): sigma = [set(self.neighbours)] result = [] while sigma: v = sigma[0].pop() - result.append(v) + yield v newsigma = [] neighb = self.neighbours[v] for s in sigma: @@ -41,8 +41,6 @@ if s2: newsigma.append(s2) sigma = newsigma - result.reverse() - return result _emptyset = frozenset() Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 13:30:39 2010 @@ -1,7 +1,7 @@ from pypy.tool.algo.color import DependencyGraph -def test_perfect_elimination_order(): +def test_lexicographic_order(): dg = DependencyGraph() dg.add_node('a') dg.add_node('b') @@ -15,8 +15,9 @@ dg.add_edge('b', 'c') dg.add_edge('b', 'e') dg.add_edge('e', 'c') - order = list(dg.perfect_elimination_order()) + order = list(dg.lexicographic_order()) assert len(order) == 5 + order.reverse() assert ''.join(order) in [ 'adbce', 'adbec', 'adcbe', 'adceb', 'adebc', 'adecb', 'acbde', 'acbed', 'acdbe', 'acdeb', 'acebd', 'acedb', From arigo at codespeak.net Wed Apr 14 13:42:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 13:42:33 +0200 (CEST) Subject: [pypy-svn] r73734 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414114233.3DEAD282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 13:42:31 2010 New Revision: 73734 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Implement size_of_largest_clique(). Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 13:42:31 2010 @@ -21,6 +21,7 @@ self.neighbours[v2].add(v1) def lexicographic_order(self): + """Enumerate a lexicographic breath-first ordering of the nodes.""" sigma = [set(self.neighbours)] result = [] while sigma: @@ -42,8 +43,20 @@ newsigma.append(s2) sigma = newsigma - -_emptyset = frozenset() + def size_of_largest_clique(self): + """Assuming that the graph is chordal, compute the size of + the largest clique in it.""" + result = 0 + seen = set() + for v in self.lexicographic_order(): + num = 1 + for n in self.neighbours[v]: + if n in seen: + num += 1 + if num > result: + result = num + seen.add(v) + return result ##class Unit(object): Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 13:42:31 2010 @@ -24,3 +24,4 @@ 'cebad', 'cebda', 'ceabd', 'ceadb', 'cedba', 'cedab', 'cabde', 'cabed', 'cadbe', 'cadeb', 'caebd', 'caedb', ] + assert dg.size_of_largest_clique() == 3 From arigo at codespeak.net Wed Apr 14 14:04:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 14:04:23 +0200 (CEST) Subject: [pypy-svn] r73735 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414120423.1A285282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 14:04:22 2010 New Revision: 73735 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Find an optimal graph coloring. The result is not optimal with respect to the number of copies potentially needed, though. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 14:04:22 2010 @@ -1,9 +1,4 @@ """ -the whole algorithm is based on the following paper: -http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.86.1578&rep=rep1&type=pdf - -and the source code of libFIRM (file ir/be/becopyheur.c): -http://pp.info.uni-karlsruhe.de/firm/Main_Page """ @@ -58,13 +53,18 @@ seen.add(v) return result - -##class Unit(object): -## """An optimization unit. Represents a phi instruction.""" -## def __init__(self, result, args): -## self.result = result -## self.args = args - -## def optimize(self, depgraph): -## self.queue = [] -## self.insert_qnode(QNode(... + def find_node_coloring(self): + """Return a random minimal node coloring, assuming that + the graph is chordal.""" + result = {} + for v in self.lexicographic_order(): + forbidden = 0 # bitset + for n in self.neighbours[v]: + if n in result: + forbidden |= (1 << result[n]) + # find the lowest 0 bit + num = 0 + while forbidden & (1 << num): + num += 1 + result[v] = num + return result Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 14:04:22 2010 @@ -25,3 +25,10 @@ 'cabde', 'cabed', 'cadbe', 'cadeb', 'caebd', 'caedb', ] assert dg.size_of_largest_clique() == 3 + coloring = dg.find_node_coloring() + assert len(coloring) == 5 + assert sorted(coloring.keys()) == list('abcde') + assert set(coloring.values()) == set([0, 1, 2]) + for v1, v2list in dg.neighbours.items(): + for v2 in v2list: + assert coloring[v1] != coloring[v2] From arigo at codespeak.net Wed Apr 14 15:21:44 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 15:21:44 +0200 (CEST) Subject: [pypy-svn] r73737 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414132144.7DF98282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 15:21:43 2010 New Revision: 73737 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Tweaks. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 15:21:43 2010 @@ -1,6 +1,3 @@ -""" -""" - class DependencyGraph(object): @@ -18,7 +15,6 @@ def lexicographic_order(self): """Enumerate a lexicographic breath-first ordering of the nodes.""" sigma = [set(self.neighbours)] - result = [] while sigma: v = sigma[0].pop() yield v @@ -55,7 +51,8 @@ def find_node_coloring(self): """Return a random minimal node coloring, assuming that - the graph is chordal.""" + the graph is chordal. For non-chordal graphs this is just + an approximately good answer (but still a valid one).""" result = {} for v in self.lexicographic_order(): forbidden = 0 # bitset Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 15:21:43 2010 @@ -1,7 +1,7 @@ from pypy.tool.algo.color import DependencyGraph -def test_lexicographic_order(): +def graph1(): dg = DependencyGraph() dg.add_node('a') dg.add_node('b') @@ -15,6 +15,10 @@ dg.add_edge('b', 'c') dg.add_edge('b', 'e') dg.add_edge('e', 'c') + return dg + +def test_lexicographic_order(): + dg = graph1() order = list(dg.lexicographic_order()) assert len(order) == 5 order.reverse() @@ -24,7 +28,13 @@ 'cebad', 'cebda', 'ceabd', 'ceadb', 'cedba', 'cedab', 'cabde', 'cabed', 'cadbe', 'cadeb', 'caebd', 'caedb', ] + +def test_size_of_largest_clique(): + dg = graph1() assert dg.size_of_largest_clique() == 3 + +def test_find_node_coloring(): + dg = graph1() coloring = dg.find_node_coloring() assert len(coloring) == 5 assert sorted(coloring.keys()) == list('abcde') From arigo at codespeak.net Wed Apr 14 16:55:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 16:55:54 +0200 (CEST) Subject: [pypy-svn] r73738 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100414145554.0978F282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 16:55:53 2010 New Revision: 73738 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py Log: Remove these lines, which stayed around as a result of some merging. It was actually moved a few lines down (so up to now, the trunk had two copies of it). Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Apr 14 16:55:53 2010 @@ -1933,11 +1933,6 @@ # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes - if self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token # is and stays 0. Note the call to reset_vable_token() in From fijal at codespeak.net Wed Apr 14 16:59:10 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 16:59:10 +0200 (CEST) Subject: [pypy-svn] r73739 - pypy/trunk/pypy/translator/jvm Message-ID: <20100414145910.7A7E2282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 16:59:08 2010 New Revision: 73739 Modified: pypy/trunk/pypy/translator/jvm/conftest.py Log: skip jvm tests on 64bit Modified: pypy/trunk/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/conftest.py (original) +++ pypy/trunk/pypy/translator/jvm/conftest.py Wed Apr 14 16:59:08 2010 @@ -1,4 +1,12 @@ +import py, sys + +class Module(py.test.collect.Module): + def collect(self): + if sys.maxint > 2147483647: # 64bit platform + py.test.skip("jvm backend on 64bit unsupported") + return super(Module, self).collect() + def pytest_addoption(parser): group = parser.getgroup("pypy-jvm options") group.addoption('--java', action='store', dest='java', default='java', From arigo at codespeak.net Wed Apr 14 17:34:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 14 Apr 2010 17:34:26 +0200 (CEST) Subject: [pypy-svn] r73740 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100414153426.EE91A282B9C@codespeak.net> Author: arigo Date: Wed Apr 14 17:34:25 2010 New Revision: 73740 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Give a deterministic result instead of a random, dict-order-dependent one. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Wed Apr 14 17:34:25 2010 @@ -2,10 +2,12 @@ class DependencyGraph(object): def __init__(self): + self.nodes = [] self.neighbours = {} def add_node(self, v): assert v not in self.neighbours, "duplicate vertex %r" % (v,) + self.nodes.append(v) self.neighbours[v] = set() def add_edge(self, v1, v2): @@ -14,20 +16,20 @@ def lexicographic_order(self): """Enumerate a lexicographic breath-first ordering of the nodes.""" - sigma = [set(self.neighbours)] + sigma = [self.nodes[:]] while sigma: v = sigma[0].pop() yield v newsigma = [] neighb = self.neighbours[v] for s in sigma: - s1 = set() - s2 = set() + s1 = [] + s2 = [] for x in s: if x in neighb: - s1.add(x) + s1.append(x) else: - s2.add(x) + s2.append(x) if s1: newsigma.append(s1) if s2: Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Wed Apr 14 17:34:25 2010 @@ -22,12 +22,10 @@ order = list(dg.lexicographic_order()) assert len(order) == 5 order.reverse() - assert ''.join(order) in [ - 'adbce', 'adbec', 'adcbe', 'adceb', 'adebc', 'adecb', - 'acbde', 'acbed', 'acdbe', 'acdeb', 'acebd', 'acedb', - 'cebad', 'cebda', 'ceabd', 'ceadb', 'cedba', 'cedab', - 'cabde', 'cabed', 'cadbe', 'cadeb', 'caebd', 'caedb', - ] + # there are many orders that are correct answers, but check that we get + # the following one, which is somehow the 'first' answer in the order + # of insertion of nodes. + assert ''.join(order) == 'acbde' def test_size_of_largest_clique(): dg = graph1() From fijal at codespeak.net Wed Apr 14 19:05:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 19:05:28 +0200 (CEST) Subject: [pypy-svn] r73742 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100414170528.4D430282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 19:05:26 2010 New Revision: 73742 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: Fix the test Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Wed Apr 14 19:05:26 2010 @@ -1,7 +1,4 @@ -import py -py.test.skip("work in progress") - from pypy.jit.tl.tinyframe import * class TestCompile(object): @@ -10,7 +7,7 @@ LOAD 0 => r1 LOAD 1 => r0 # comment # other comment - ADD r0, r1 => r2 + ADD r0 r1 => r2 PRINT r2 ''') assert disassemble(code) == [ Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Wed Apr 14 19:05:26 2010 @@ -4,7 +4,7 @@ register based with untyped registers. Opcodes: -ADD r1, r2 => r3 # integer addition or function combination, +ADD r1 r2 => r3 # integer addition or function combination, # depending on argument types # if r1 has a function f and r2 has a function g # the result will be a function lambda arg : f(g(arg)) @@ -28,6 +28,10 @@ def __init__(self, code): code = code +def rint(arg): + assert arg.startswith('r') + return int(arg[1:]) + class Parser(object): def compile(self, strrepr): self.code = [] @@ -40,13 +44,25 @@ continue opcode, args = line.split(" ", 1) getattr(self, 'compile_' + opcode)(args) + return "".join([chr(i) for i in self.code]) def compile_ADD(self, args): - xxx + args, result = args.split("=>") + arg0, arg1 = args.strip().split(" ") + self.code += [ADD, rint(arg0), rint(arg1), rint(result.strip())] def compile_LOAD(self, args): - self.code.append( + arg0, result = args.split("=>") + arg0 = arg0.strip() + self.code += [LOAD, int(arg0), rint(result.strip())] + + def compile_PRINT(self, args): + arg = rint(args.strip()) + self.code += [PRINT, arg] def compile(strrepr): parser = Parser() return parser.compile(strrepr) + +def disassemble(code): + return [ord(i) for i in code] From afa at codespeak.net Wed Apr 14 19:30:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 14 Apr 2010 19:30:31 +0200 (CEST) Subject: [pypy-svn] r73743 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100414173031.4FE17282B9C@codespeak.net> Author: afa Date: Wed Apr 14 19:30:29 2010 New Revision: 73743 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Start refactoring the description of the various Py...Object types. The goal is to remove the special cases in almost every important operation, still in progress! Every type calls make_typedescr() to specify how objects are allocated, freed, converted from/to the main interpreter. --Cette ligne, et les suivantes ci-dessous, seront ignor?es-- M cpyext/stringobject.py M cpyext/__init__.py M cpyext/api.py M cpyext/typeobjectdefs.py M cpyext/unicodeobject.py M cpyext/pycobject.py M cpyext/test/test_stringobject.py M cpyext/typeobject.py M cpyext/pyobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Wed Apr 14 19:30:29 2010 @@ -32,7 +32,7 @@ state.init_r2w_from_w2r() for func in api.INIT_FUNCTIONS: - func() + func(space) state.check_and_raise_exception() if not we_are_translated(): state.non_heaptypes[:] = [] Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 14 19:30:29 2010 @@ -249,6 +249,7 @@ '_Py_NotImplementedStruct#': ('PyObject*', 'space.w_NotImplemented'), } INIT_FUNCTIONS = [] +BOOTSTRAP_FUNCTIONS = [] for exc_name in exceptions.Module.interpleveldefs.keys(): GLOBALS['PyExc_' + exc_name] = ('PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) @@ -301,18 +302,6 @@ # a pointer to PyObject PyObjectP = rffi.CArrayPtr(PyObject) -PyStringObjectStruct = lltype.ForwardReference() -PyStringObject = lltype.Ptr(PyStringObjectStruct) -PyStringObjectFields = PyObjectFields + \ - (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) -cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) - -PyUnicodeObjectStruct = lltype.ForwardReference() -PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) -PyUnicodeObjectFields = (PyObjectFields + - (("buffer", rffi.CWCHARP), ("size", Py_ssize_t))) -cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) - VA_TP_LIST = {} #{'int': lltype.Signed, # 'PyObject*': PyObject, @@ -411,6 +400,9 @@ except NullPointerException: print "Container not registered by %s" % callable.__name__ except Exception: + if not we_are_translated(): + import traceback + traceback.print_exc() print "Fatal exception encountered" rffi.stackcounter.stacks_counter -= 1 return retval @@ -429,17 +421,31 @@ globals()['va_get_%s' % name_no_star] = func def setup_init_functions(eci): + init_buffer = rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) INIT_FUNCTIONS.extend([ - rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) + lambda space: init_buffer(), ]) +def init_function(func): + INIT_FUNCTIONS.append(func) + return func + +def bootstrap_function(func): + BOOTSTRAP_FUNCTIONS.append(func) + return func + def bootstrap_types(space): from pypy.module.cpyext.pyobject import make_ref from pypy.module.cpyext.typeobject import PyTypeObjectPtr, PyPyType_Ready, \ inherit_slots + + for func in BOOTSTRAP_FUNCTIONS: + func(space) + # bootstrap this damn cycle type_pto = make_ref(space, space.w_type) type_pto = rffi.cast(PyTypeObjectPtr, type_pto) + object_pto = make_ref(space, space.w_object) object_pto = rffi.cast(PyTypeObjectPtr, object_pto) type_pto.c_tp_base = object_pto Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Wed Apr 14 19:30:29 2010 @@ -1,9 +1,9 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.pyobject import make_ref -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ - cpython_struct, PyObjectFields +from pypy.module.cpyext.pyobject import make_ref, make_typedescr +from pypy.module.cpyext.api import generic_cpy_call, cpython_api, bootstrap_function, \ + PyObject, cpython_struct, PyObjectFields destructor_short = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) @@ -16,6 +16,16 @@ def __init__(self, space): self.space = space +W_PyCObject.typedef = TypeDef( + 'PyCObject', + ) +W_PyCObject.typedef.acceptable_as_base_class = False + + at bootstrap_function +def init_pycobject(space): + make_typedescr(W_PyCObject.typedef, + basestruct=PyCObjectStruct) + class W_PyCObjectFromVoidPtr(W_PyCObject): def __init__(self, space, voidp, desc): W_PyCObject.__init__(self, space) @@ -64,9 +74,3 @@ return pyo -W_PyCObject.typedef = TypeDef( - 'PyCObject', - ) -W_PyCObject.typedef.acceptable_as_base_class = False - - Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Wed Apr 14 19:30:29 2010 @@ -1,13 +1,89 @@ import sys -from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.baseobjspace import W_Root, Wrappable, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, PyObject, PyStringObject, ADDR,\ - Py_TPFLAGS_HEAPTYPE, PyUnicodeObject, PyTypeObjectPtr +from pypy.module.cpyext.api import cpython_api, \ + PyObject, ADDR,\ + Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.unicodeobject import W_UnicodeObject -from pypy.rlib.objectmodel import we_are_translated +from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.objectobject import W_ObjectObject +from pypy.rlib.objectmodel import specialize, we_are_translated +from pypy.rpython.annlowlevel import llhelper + +#________________________________________________________ +# type description + +class BaseCpyTypedescr(object): + pass + +typedescr_cache = {} + +def make_typedescr(typedef, **kw): + """NOT_RPYTHON + + basestruct: The basic structure to allocate + realize : Function called to create a pypy object from a raw struct + dealloc : a cpython_api(external=False), similar to PyObject_dealloc + """ + + tp_basestruct = kw.get('basestruct', PyObject.TO) + tp_realize = kw.get('realize') + tp_dealloc = kw.get('dealloc') + + null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void)) + + class CpyTypedescr(BaseCpyTypedescr): + basestruct = tp_basestruct + realize = tp_realize + + def get_dealloc(self, space): + if tp_dealloc: + return llhelper( + tp_dealloc.api_func.functype, + tp_dealloc.api_func.get_wrapper(space)) + else: + return null_dealloc # XXX PyObject_dealloc? + + def allocate(self, space, w_type): + # similar to PyType_GenericAlloc? + # except that it's not related to any pypy object. + obj = lltype.malloc(tp_basestruct, flavor='raw', zero=True) + pyobj = rffi.cast(PyObject, obj) + pyobj.c_ob_refcnt = 1 + if w_type is not space.w_type: + pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + return pyobj + + if tp_realize: + def realize(self, space, ref): + return tp_realize(space, ref) + else: + def realize(self, space, ref): + raise TypeError("cannot realize") + if typedef: + CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,) + + typedescr_cache[typedef] = CpyTypedescr() + +make_typedescr(None) + + at specialize.memo() +def _get_typedescr_1(typedef): + try: + return typedescr_cache[typedef] + except KeyError: + if typedef.base is not None: + return _get_typedescr_1(typedef.base) + return typedescr_cache[None] + +def get_typedescr(typedef): + if typedef is None: + return typedescr_cache[None] + else: + return _get_typedescr_1(typedef) #________________________________________________________ # refcounted object support @@ -29,11 +105,60 @@ print >>sys.stderr, arg, print >>sys.stderr - -def make_ref(space, w_obj, borrowed=False, steal=False, items=0): +def create_ref(space, w_obj, items=0): from pypy.module.cpyext.typeobject import allocate_type_obj,\ W_PyCTypeObject, PyOLifeline from pypy.module.cpyext.pycobject import W_PyCObject, PyCObject + w_type = space.type(w_obj) + if space.is_w(w_type, space.w_type) or space.is_w(w_type, + space.gettypeobject(W_PyCTypeObject.typedef)): + py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + + # put the type object early into the dict + # to support dependency cycles like object/type + state = space.fromcache(State) + state.py_objects_w2r[w_obj] = py_obj + py_obj.c_ob_type = rffi.cast( + PyTypeObjectPtr, make_ref(space, w_type, + steal=not space.is_w(w_type, space.w_type))) + + allocate_type_obj(space, py_obj, w_obj) + elif isinstance(w_type, W_PyCTypeObject): + lifeline = w_obj.get_pyolifeline() + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + Py_IncRef(space, py_obj) + else: + w_type_pyo = make_ref(space, w_type) + pto = rffi.cast(PyTypeObjectPtr, w_type_pyo) + # Don't increase refcount for non-heaptypes + if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, w_type_pyo) + basicsize = pto.c_tp_basicsize + items * pto.c_tp_itemsize + py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, + flavor="raw", zero=True) + py_obj = rffi.cast(PyObject, py_obj_pad) + py_obj.c_ob_refcnt = 1 + py_obj.c_ob_type = pto + w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) + elif isinstance(w_obj, W_StringObject): + py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + from pypy.module.cpyext.stringobject import PyStringObject + py_str = rffi.cast(PyStringObject, py_obj) + py_str.c_size = len(space.str_w(w_obj)) + py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + elif isinstance(w_obj, W_UnicodeObject): + py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + from pypy.module.cpyext.unicodeobject import PyUnicodeObject + py_unicode = rffi.cast(PyUnicodeObject, py_obj) + py_unicode.c_size = len(space.unicode_w(w_obj)) + py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) + else: + py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + return py_obj + +def make_ref(space, w_obj, borrowed=False, steal=False, items=0): if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) @@ -41,58 +166,7 @@ py_obj = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not py_obj: assert not steal - w_type = space.type(w_obj) - if space.is_w(w_type, space.w_type) or space.is_w(w_type, - space.gettypeobject(W_PyCTypeObject.typedef)): - pto = allocate_type_obj(space, w_obj) - py_obj = rffi.cast(PyObject, pto) - # c_ob_type and c_ob_refcnt are set by allocate_type_obj - elif isinstance(w_type, W_PyCTypeObject): - lifeline = w_obj.get_pyolifeline() - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - Py_IncRef(space, py_obj) - else: - w_type_pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, w_type_pyo) - # Don't increase refcount for non-heaptypes - if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, w_type_pyo) - basicsize = pto.c_tp_basicsize + items * pto.c_tp_itemsize - py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, - flavor="raw", zero=True) - py_obj = rffi.cast(PyObject, py_obj_pad) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = pto - w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) - elif isinstance(w_obj, W_StringObject): - py_obj_str = lltype.malloc(PyStringObject.TO, flavor='raw', zero=True) - py_obj_str.c_size = len(space.str_w(w_obj)) - py_obj_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) - pto = make_ref(space, space.w_str) - py_obj = rffi.cast(PyObject, py_obj_str) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) - elif isinstance(w_obj, W_UnicodeObject): - py_obj_unicode = lltype.malloc(PyUnicodeObject.TO, flavor='raw', zero=True) - py_obj_unicode.c_size = len(space.unicode_w(w_obj)) - py_obj_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) - pto = make_ref(space, space.w_unicode) - py_obj = rffi.cast(PyObject, py_obj_unicode) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) - elif isinstance(w_obj, W_PyCObject): # a PyCObject - py_cobj = lltype.malloc(PyCObject.TO, flavor='raw', zero=True) - pto = make_ref(space, space.type(w_obj)) - py_obj = rffi.cast(PyObject, py_cobj) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) - else: - py_obj = lltype.malloc(PyObject.TO, flavor="raw", zero=True) - py_obj.c_ob_refcnt = 1 - pto = make_ref(space, space.type(w_obj)) - py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, pto) + py_obj = create_ref(space, w_obj, items) ptr = rffi.cast(ADDR, py_obj) if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) @@ -110,17 +184,6 @@ Py_IncRef(space, py_obj) return py_obj -def force_string(space, ref): - state = space.fromcache(State) - ref = rffi.cast(PyStringObject, ref) - s = rffi.charpsize2str(ref.c_buffer, ref.c_size) - ref = rffi.cast(PyObject, ref) - w_str = space.wrap(s) - state.py_objects_w2r[w_str] = ref - ptr = rffi.cast(ADDR, ref) - state.py_objects_r2w[ptr] = w_str - return w_str - def from_ref(space, ref, recurse=False): from pypy.module.cpyext.typeobject import PyPyType_Ready @@ -137,8 +200,9 @@ ref_type = rffi.cast(PyObject, ref.c_ob_type) if ref != ref_type: w_type = from_ref(space, ref_type, True) + assert isinstance(w_type, W_TypeObject) if space.is_w(w_type, space.w_str): - return force_string(space, ref) + return get_typedescr(w_type.instancetypedef).realize(space, ref) elif space.is_w(w_type, space.w_type): PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) return from_ref(space, ref, True) @@ -156,7 +220,7 @@ return assert lltype.typeOf(obj) == PyObject - from pypy.module.cpyext.typeobject import string_dealloc, W_PyCTypeObject + from pypy.module.cpyext.typeobject import W_PyCTypeObject obj.c_ob_refcnt -= 1 if DEBUG_REFCOUNT: debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Wed Apr 14 19:30:29 2010 @@ -1,11 +1,24 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyVarObjectFields, - PyStringObject, Py_ssize_t, cpython_struct, - CANNOT_FAIL, build_type_checkers, - PyObjectP, PyTypeObjectPtr) -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef - +from pypy.module.cpyext.api import (cpython_api, bootstrap_function, PyVarObjectFields, + Py_ssize_t, cpython_struct, PyObjectFields, + ADDR, CANNOT_FAIL, build_type_checkers, + PyObjectP, PyTypeObjectPtr, generic_cpy_call) +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr +from pypy.module.cpyext.state import State + +PyStringObjectStruct = lltype.ForwardReference() +PyStringObject = lltype.Ptr(PyStringObjectStruct) +PyStringObjectFields = PyObjectFields + \ + (("buffer", rffi.CCHARP), ("size", Py_ssize_t)) +cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) + + at bootstrap_function +def init_stringobject(space): + make_typedescr(space.w_str.instancetypedef, + basestruct=PyStringObject.TO, + dealloc=string_dealloc, + realize=string_realize) PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") @@ -20,6 +33,28 @@ py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str +def string_realize(space, ref): + state = space.fromcache(State) + ref = rffi.cast(PyStringObject, ref) + s = rffi.charpsize2str(ref.c_buffer, ref.c_size) + ref = rffi.cast(PyObject, ref) + w_str = space.wrap(s) + state.py_objects_w2r[w_str] = ref + ptr = rffi.cast(ADDR, ref) + state.py_objects_r2w[ptr] = w_str + return w_str + + at cpython_api([PyObject], lltype.Void, external=False) +def string_dealloc(space, obj): + obj = rffi.cast(PyStringObject, obj) + pto = obj.c_ob_type + if obj.c_buffer: + lltype.free(obj.c_buffer, flavor="raw") + obj_voidp = rffi.cast(rffi.VOIDP_real, obj) + generic_cpy_call(space, pto.c_tp_free, obj_voidp) + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) + @cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): if char_p: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Wed Apr 14 19:30:29 2010 @@ -1,8 +1,8 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.stringobject import new_empty_str -from pypy.module.cpyext.api import PyStringObject, PyObjectP, PyObject +from pypy.module.cpyext.stringobject import new_empty_str, PyStringObject +from pypy.module.cpyext.api import PyObjectP, PyObject from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref import py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 14 19:30:29 2010 @@ -12,12 +12,12 @@ from pypy.objspace.std.typetype import _precheck_for_new from pypy.objspace.std.objectobject import W_ObjectObject from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.cpyext.api import cpython_api, cpython_struct, \ +from pypy.module.cpyext.api import cpython_api, cpython_struct, bootstrap_function, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - PyUnicodeObject, CANNOT_FAIL, PyBufferProcs -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref + CANNOT_FAIL, PyBufferProcs +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, get_typedescr, make_typedescr from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -276,6 +276,12 @@ check_descr(space, w_self, self.pto) PyMember_SetOne(space, w_self, self.member, w_value) + at bootstrap_function +def init_unicodeobject(space): + make_typedescr(space.w_type.instancetypedef, + basestruct=PyTypeObject, + dealloc=type_dealloc) + def c_type_descr__call__(space, w_type, __args__): if isinstance(w_type, W_PyCTypeObject): pyo = make_ref(space, w_type) @@ -359,28 +365,6 @@ pto.c_tp_as_buffer = c_buf @cpython_api([PyObject], lltype.Void, external=False) -def string_dealloc(space, obj): - obj = rffi.cast(PyStringObject, obj) - pto = obj.c_ob_type - if obj.c_buffer: - lltype.free(obj.c_buffer, flavor="raw") - obj_voidp = rffi.cast(rffi.VOIDP_real, obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) - - at cpython_api([PyObject], lltype.Void, external=False) -def unicode_dealloc(space, obj): - obj = rffi.cast(PyUnicodeObject, obj) - pto = obj.c_ob_type - if obj.c_buffer: - lltype.free(obj.c_buffer, flavor="raw") - obj_voidp = rffi.cast(rffi.VOIDP_real, obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) - - at cpython_api([PyObject], lltype.Void, external=False) def type_dealloc(space, obj): state = space.fromcache(State) obj_pto = rffi.cast(PyTypeObjectPtr, obj) @@ -399,36 +383,33 @@ Py_DecRef(space, pto) -def allocate_type_obj(space, w_type): - """ Allocates a pto from a w_type which must be a PyPy type. """ - state = space.fromcache(State) +def allocate_type_obj(space, py_obj, w_type): + """ Allocates a PyTypeObject from a w_type which must be a PyPy type. """ from pypy.module.cpyext.object import PyObject_dealloc, PyObject_Del + assert isinstance(w_type, W_TypeObject) - pto = lltype.malloc(PyTypeObject, flavor="raw", zero=True) - pto.c_ob_refcnt = 1 - # put the type object early into the dict - # to support dependency cycles like object/type - state = space.fromcache(State) - state.py_objects_w2r[w_type] = rffi.cast(PyObject, pto) + pto = rffi.cast(PyTypeObjectPtr, py_obj) + # dealloc if space.is_w(w_type, space.w_object): pto.c_tp_dealloc = llhelper(PyObject_dealloc.api_func.functype, PyObject_dealloc.api_func.get_wrapper(space)) elif space.is_w(w_type, space.w_type): pto.c_tp_dealloc = llhelper(type_dealloc.api_func.functype, type_dealloc.api_func.get_wrapper(space)) + #pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) elif space.is_w(w_type, space.w_str): - pto.c_tp_dealloc = llhelper(string_dealloc.api_func.functype, - string_dealloc.api_func.get_wrapper(space)) - # buffer protocol - setup_string_buffer_procs(space, pto) + pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) elif space.is_w(w_type, space.w_unicode): - pto.c_tp_dealloc = llhelper(unicode_dealloc.api_func.functype, - unicode_dealloc.api_func.get_wrapper(space)) + pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) else: pto.c_tp_dealloc = llhelper(subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) + # buffer protocol + if space.is_w(w_type, space.w_str): + setup_string_buffer_procs(space, pto) + pto.c_tp_flags = Py_TPFLAGS_HEAPTYPE pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, PyObject_Del.api_func.get_wrapper(space)) @@ -448,16 +429,13 @@ if bases_w: ref = make_ref(space, bases_w[0]) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) - pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.type(space.w_type))) PyPyType_Ready(space, pto, w_type) - else: - pto.c_ob_type = lltype.nullptr(PyTypeObjectPtr.TO) if space.is_w(w_type, space.w_object): pto.c_tp_basicsize = rffi.sizeof(PyObject.TO) elif space.is_w(w_type, space.w_type): pto.c_tp_basicsize = rffi.sizeof(PyTypeObject) elif space.is_w(w_type, space.w_str): - pto.c_tp_basicsize = rffi.sizeof(PyStringObject.TO) + pto.c_tp_basicsize = rffi.sizeof(get_typedescr(w_type.instancetypedef).basestruct) elif pto.c_tp_base: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobjectdefs.py Wed Apr 14 19:30:29 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.lltype import Ptr, FuncType, Void from pypy.module.cpyext.api import cpython_struct, \ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, \ - Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, PyStringObject, ADDR, \ + Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ PyTypeObject, PyTypeObjectPtr, PyBufferProcs from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 14 19:30:29 2010 @@ -1,14 +1,27 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb -from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, PyUnicodeObject, - build_type_checkers, cpython_api) +from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, + build_type_checkers, cpython_api, + bootstrap_function, generic_cpy_call) from pypy.module.cpyext.pyerrors import PyErr_BadArgument -from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref +from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr from pypy.module.cpyext.stringobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype +PyUnicodeObjectStruct = lltype.ForwardReference() +PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) +PyUnicodeObjectFields = (PyObjectFields + + (("buffer", rffi.CWCHARP), ("size", Py_ssize_t))) +cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) + + at bootstrap_function +def init_unicodeobject(space): + make_typedescr(space.w_unicode.instancetypedef, + basestruct=PyUnicodeObject.TO, + dealloc=unicode_dealloc) + # Buffer for the default encoding (used by PyUnicde_GetDefaultEncoding) DEFAULT_ENCODING_SIZE = 100 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE, @@ -18,6 +31,17 @@ Py_UNICODE = lltype.UniChar + at cpython_api([PyObject], lltype.Void, external=False) +def unicode_dealloc(space, obj): + obj = rffi.cast(PyUnicodeObject, obj) + pto = obj.c_ob_type + if obj.c_buffer: + lltype.free(obj.c_buffer, flavor="raw") + obj_voidp = rffi.cast(rffi.VOIDP_real, obj) + generic_cpy_call(space, pto.c_tp_free, obj_voidp) + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) + @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): """Return 1 or 0 depending on whether ch is a whitespace character.""" From fijal at codespeak.net Wed Apr 14 20:58:34 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 20:58:34 +0200 (CEST) Subject: [pypy-svn] r73744 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100414185834.3EF54282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 20:58:32 2010 New Revision: 73744 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: pass another test Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Wed Apr 14 20:58:32 2010 @@ -13,3 +13,13 @@ assert disassemble(code) == [ LOAD, 0, 1, LOAD, 1, 0, ADD, 0, 1, 2, PRINT, 2 ] + + def test_return(self): + code = compile(''' + LOAD 0 => r1 + LOAD 1 => r0 # comment + # other comment + ADD r0 r1 => r2 + RETURN r2 + ''') + res = interpret(code) Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Wed Apr 14 20:58:32 2010 @@ -14,27 +14,25 @@ CALL r1 r2 # call a function in register one with argument in r2 LOAD => r # load a function named name into register r LOAD => r # load an integer constant into register r +RETURN r1 function argument always comes in r0 """ -opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD'] +opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'RETURN'] for i, opcode in enumerate(opcodes): globals()[opcode] = i class Code(object): - _immutable_ = True - - def __init__(self, code): - code = code - -def rint(arg): - assert arg.startswith('r') - return int(arg[1:]) + def __init__(self, code, regno): + self.code = code + self.regno = regno class Parser(object): + def compile(self, strrepr): self.code = [] + self.maxregno = 0 for line in strrepr.splitlines(): comment = line.find('#') if comment != -1: @@ -44,25 +42,76 @@ continue opcode, args = line.split(" ", 1) getattr(self, 'compile_' + opcode)(args) - return "".join([chr(i) for i in self.code]) + return Code("".join([chr(i) for i in self.code]), self.maxregno + 1) + + def rint(self, arg): + assert arg.startswith('r') + no = int(arg[1:]) + self.maxregno = max(self.maxregno, no) + return no def compile_ADD(self, args): args, result = args.split("=>") arg0, arg1 = args.strip().split(" ") - self.code += [ADD, rint(arg0), rint(arg1), rint(result.strip())] + self.code += [ADD, self.rint(arg0), self.rint(arg1), + self.rint(result.strip())] def compile_LOAD(self, args): arg0, result = args.split("=>") arg0 = arg0.strip() - self.code += [LOAD, int(arg0), rint(result.strip())] + self.code += [LOAD, int(arg0), self.rint(result.strip())] def compile_PRINT(self, args): - arg = rint(args.strip()) + arg = self.rint(args.strip()) self.code += [PRINT, arg] + def compile_RETURN(self, args): + arg = self.rint(args.strip()) + self.code += [RETURN, arg] + def compile(strrepr): parser = Parser() return parser.compile(strrepr) def disassemble(code): - return [ord(i) for i in code] + return [ord(i) for i in code.code] + +class Object(object): + def __init__(self): + raise NotImplementedError("abstract base class") + + def add(self, other): + raise NotImplementedError("abstract base class") + +class Int(Object): + def __init__(self, val): + self.val = val + + def add(self, other): + return Int(self.val + other.val) + +class Frame(object): + def __init__(self, code): + self.code = code + self.registers = [None] * code.regno + + def interpret(self): + i = 0 + code = self.code.code + while True: + opcode = ord(code[i]) + if opcode == LOAD: + self.registers[ord(code[i + 2])] = Int(ord(code[i + 1])) + i += 3 + elif opcode == ADD: + arg1 = self.registers[ord(code[i + 1])] + arg2 = self.registers[ord(code[i + 2])] + self.registers[ord(code[i + 3])] = arg1.add(arg2) + i += 4 + elif opcode == RETURN: + return self.registers[ord(code[i + 1])] + else: + raise Exception("unimplemented opcode %s" % opcodes[opcode]) + +def interpret(code): + return Frame(code).interpret() From fijal at codespeak.net Wed Apr 14 21:46:43 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 21:46:43 +0200 (CEST) Subject: [pypy-svn] r73745 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100414194643.28944282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 21:46:41 2010 New Revision: 73745 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: Add loops Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Wed Apr 14 21:46:41 2010 @@ -1,9 +1,11 @@ +import py from pypy.jit.tl.tinyframe import * class TestCompile(object): def test_simple(self): code = compile(''' + main: LOAD 0 => r1 LOAD 1 => r0 # comment # other comment @@ -16,6 +18,7 @@ def test_return(self): code = compile(''' + main: LOAD 0 => r1 LOAD 1 => r0 # comment # other comment @@ -23,3 +26,31 @@ RETURN r2 ''') res = interpret(code) + assert isinstance(res, Int) + assert res.val == 1 + + def test_loop(self): + code = compile(''' + main: + LOAD 1 => r1 + LOAD 100 => r2 + LOAD 0 => r0 + @l1 + ADD r0 r1 => r0 + JUMP_IF_ABOVE r2 r0 @l1 + RETURN r0 + ''') + ret = interpret(code) + assert ret.val == 100 + + def test_function(self): + py.test.skip("not yet") + code = compile(''' + func: # arg comes in r0 + LOAD 1 => r1 + ADD r0 r1 => r1 + RETURN r1 + main: + LOAD_FUNCTION name + CALL + ''') Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Wed Apr 14 21:46:41 2010 @@ -12,37 +12,70 @@ # pointed by r1 (must be int) to r2 PRINT r # print a register CALL r1 r2 # call a function in register one with argument in r2 -LOAD => r # load a function named name into register r +LOAD_FUNCTION => r # load a function named name into register r LOAD => r # load an integer constant into register r RETURN r1 +JUMP @label # jump + or - by x opcodes +JUMP_IF_ABOVE r1 r2 @label # jump if value in r1 is above +# value in r2 function argument always comes in r0 """ -opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'RETURN'] +opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', + 'RETURN', 'JUMP', 'JUMP_IF_ABOVE'] for i, opcode in enumerate(opcodes): globals()[opcode] = i class Code(object): - def __init__(self, code, regno): + def __init__(self, code, regno, functions): self.code = code self.regno = regno + self.functions = functions class Parser(object): + + name = None def compile(self, strrepr): self.code = [] self.maxregno = 0 - for line in strrepr.splitlines(): + self.functions = {} + self.labels = {} + lines = strrepr.splitlines() + for line in lines: comment = line.find('#') if comment != -1: line = line[:comment] line = line.strip() if not line: continue + if line.endswith(':'): + # a name + self.finish_currect_code() + self.name = line[:-1] + continue + if line.startswith('@'): + self.labels[line[1:]] = len(self.code) + continue opcode, args = line.split(" ", 1) getattr(self, 'compile_' + opcode)(args) - return Code("".join([chr(i) for i in self.code]), self.maxregno + 1) + functions = [code for i, code in sorted(self.functions.values())] + assert self.name == 'main' + return Code("".join([chr(i) for i in self.code]), self.maxregno + 1, + functions) + + def finish_currect_code(self): + if self.name is None: + assert not self.code + return + code = Code("".join([chr(i) for i in self.code]), self.maxregno + 1, + []) + self.functions[self.name] = (len(self.functions), code) + self.name = None + self.labels = {} + self.code = [] + self.maxregno = 0 def rint(self, arg): assert arg.startswith('r') @@ -69,6 +102,11 @@ arg = self.rint(args.strip()) self.code += [RETURN, arg] + def compile_JUMP_IF_ABOVE(self, args): + arg0, arg1, label = args.split(" ") + self.code += [JUMP_IF_ABOVE, self.rint(arg0.strip()), + self.rint(arg1.strip()), self.labels[label[1:]]] + def compile(strrepr): parser = Parser() return parser.compile(strrepr) @@ -83,6 +121,9 @@ def add(self, other): raise NotImplementedError("abstract base class") + def gt(self, other): + raise NotImplementedError("abstract base class") + class Int(Object): def __init__(self, val): self.val = val @@ -90,6 +131,9 @@ def add(self, other): return Int(self.val + other.val) + def gt(self, other): + return self.val > other.val + class Frame(object): def __init__(self, code): self.code = code @@ -110,6 +154,14 @@ i += 4 elif opcode == RETURN: return self.registers[ord(code[i + 1])] + elif opcode == JUMP_IF_ABOVE: + arg0 = self.registers[ord(code[i + 1])] + arg1 = self.registers[ord(code[i + 2])] + tgt = ord(code[i + 3]) + if arg0.gt(arg1): + i = tgt + else: + i += 4 else: raise Exception("unimplemented opcode %s" % opcodes[opcode]) From benjamin at codespeak.net Wed Apr 14 23:20:38 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 14 Apr 2010 23:20:38 +0200 (CEST) Subject: [pypy-svn] r73747 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414212038.51202282B9C@codespeak.net> Author: benjamin Date: Wed Apr 14 23:20:36 2010 New Revision: 73747 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: add imports Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 14 23:20:36 2010 @@ -3,7 +3,8 @@ from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, - bootstrap_function, generic_cpy_call) + bootstrap_function, generic_cpy_call, + PyObjectFields, cpython_struct) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr from pypy.module.cpyext.stringobject import PyString_Check From fijal at codespeak.net Wed Apr 14 23:29:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 14 Apr 2010 23:29:35 +0200 (CEST) Subject: [pypy-svn] r73748 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100414212935.BD2E2282B9C@codespeak.net> Author: fijal Date: Wed Apr 14 23:29:34 2010 New Revision: 73748 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: IN-PROGRESS Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Wed Apr 14 23:29:34 2010 @@ -44,13 +44,17 @@ assert ret.val == 100 def test_function(self): - py.test.skip("not yet") + py.test.skip("in progress") code = compile(''' func: # arg comes in r0 LOAD 1 => r1 ADD r0 r1 => r1 RETURN r1 main: - LOAD_FUNCTION name - CALL + LOAD_FUNCTION func => r0 + LOAD 1 => r1 + CALL r0 r1 => r2 + RETURN r2 ''') + ret = interpret(code) + assert ret.val == 1 + 1 Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Wed Apr 14 23:29:34 2010 @@ -8,10 +8,12 @@ # depending on argument types # if r1 has a function f and r2 has a function g # the result will be a function lambda arg : f(g(arg)) + # this is also a way to achieve indirect call INTROSPECT r1 => r2 # frame introspection - load a register with number # pointed by r1 (must be int) to r2 PRINT r # print a register -CALL r1 r2 # call a function in register one with argument in r2 +CALL r1 r2 => r3 # call a function in register one with argument in r2 and + # result in r3 LOAD_FUNCTION => r # load a function named name into register r LOAD => r # load an integer constant into register r RETURN r1 @@ -107,6 +109,8 @@ self.code += [JUMP_IF_ABOVE, self.rint(arg0.strip()), self.rint(arg1.strip()), self.labels[label[1:]]] + #def compile_LOAD_FUNCTION(self, + def compile(strrepr): parser = Parser() return parser.compile(strrepr) From xoraxax at codespeak.net Thu Apr 15 00:28:25 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Thu, 15 Apr 2010 00:28:25 +0200 (CEST) Subject: [pypy-svn] r73749 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414222825.A3426282B9C@codespeak.net> Author: xoraxax Date: Thu Apr 15 00:28:24 2010 New Revision: 73749 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Update TODO. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Thu Apr 15 00:28:24 2010 @@ -15,7 +15,7 @@ - Fix distutil's build_ext to work with cpyext on windows. - - Fix GIL handling (e.g. init is called without GIL held). + - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. From afa at codespeak.net Thu Apr 15 00:45:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Apr 2010 00:45:56 +0200 (CEST) Subject: [pypy-svn] r73750 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414224556.E489F282B9C@codespeak.net> Author: afa Date: Thu Apr 15 00:45:54 2010 New Revision: 73750 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: typedescr.attach() fills a PyObject-based structure from w_obj. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 15 00:45:54 2010 @@ -6,8 +6,6 @@ PyObject, ADDR,\ Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State -from pypy.objspace.std.stringobject import W_StringObject -from pypy.objspace.std.unicodeobject import W_UnicodeObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.rlib.objectmodel import specialize, we_are_translated @@ -25,11 +23,15 @@ """NOT_RPYTHON basestruct: The basic structure to allocate + alloc : allocate and basic initialization of a raw PyObject + attach : Function called to tie a raw structure to a pypy object realize : Function called to create a pypy object from a raw struct dealloc : a cpython_api(external=False), similar to PyObject_dealloc """ tp_basestruct = kw.get('basestruct', PyObject.TO) + tp_alloc = kw.get('alloc') + tp_attach = kw.get('attach') tp_realize = kw.get('realize') tp_dealloc = kw.get('dealloc') @@ -47,15 +49,26 @@ else: return null_dealloc # XXX PyObject_dealloc? - def allocate(self, space, w_type): - # similar to PyType_GenericAlloc? - # except that it's not related to any pypy object. - obj = lltype.malloc(tp_basestruct, flavor='raw', zero=True) - pyobj = rffi.cast(PyObject, obj) - pyobj.c_ob_refcnt = 1 - if w_type is not space.w_type: - pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) - return pyobj + if tp_alloc: + def allocate(self, space, w_type): + return tp_alloc(space, w_type) + else: + def allocate(self, space, w_type): + # similar to PyType_GenericAlloc? + # except that it's not related to any pypy object. + obj = lltype.malloc(tp_basestruct, flavor='raw', zero=True) + pyobj = rffi.cast(PyObject, obj) + pyobj.c_ob_refcnt = 1 + if w_type is not space.w_type: + pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + return pyobj + + if tp_attach: + def attach(self, space, pyobj, w_obj): + tp_attach(space, pyobj, w_obj) + else: + def attach(self, space, pyobj, w_obj): + pass if tp_realize: def realize(self, space, ref): @@ -106,13 +119,14 @@ print >>sys.stderr def create_ref(space, w_obj, items=0): - from pypy.module.cpyext.typeobject import allocate_type_obj,\ - W_PyCTypeObject, PyOLifeline + from pypy.module.cpyext.typeobject import W_PyCTypeObject, PyOLifeline from pypy.module.cpyext.pycobject import W_PyCObject, PyCObject w_type = space.type(w_obj) + typedescr = get_typedescr(w_obj.typedef) + if space.is_w(w_type, space.w_type) or space.is_w(w_type, space.gettypeobject(W_PyCTypeObject.typedef)): - py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + py_obj = typedescr.allocate(space, w_type) # put the type object early into the dict # to support dependency cycles like object/type @@ -122,7 +136,7 @@ PyTypeObjectPtr, make_ref(space, w_type, steal=not space.is_w(w_type, space.w_type))) - allocate_type_obj(space, py_obj, w_obj) + typedescr.attach(space, py_obj, w_obj) elif isinstance(w_type, W_PyCTypeObject): lifeline = w_obj.get_pyolifeline() if lifeline is not None: # make old PyObject ready for use in C code @@ -142,20 +156,9 @@ py_obj.c_ob_refcnt = 1 py_obj.c_ob_type = pto w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) - elif isinstance(w_obj, W_StringObject): - py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) - from pypy.module.cpyext.stringobject import PyStringObject - py_str = rffi.cast(PyStringObject, py_obj) - py_str.c_size = len(space.str_w(w_obj)) - py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) - elif isinstance(w_obj, W_UnicodeObject): - py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) - from pypy.module.cpyext.unicodeobject import PyUnicodeObject - py_unicode = rffi.cast(PyUnicodeObject, py_obj) - py_unicode.c_size = len(space.unicode_w(w_obj)) - py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) else: py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + typedescr.attach(space, py_obj, w_obj) return py_obj def make_ref(space, w_obj, borrowed=False, steal=False, items=0): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Thu Apr 15 00:45:54 2010 @@ -17,6 +17,7 @@ def init_stringobject(space): make_typedescr(space.w_str.instancetypedef, basestruct=PyStringObject.TO, + attach=string_attach, dealloc=string_dealloc, realize=string_realize) @@ -33,6 +34,11 @@ py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str +def string_attach(space, py_obj, w_obj): + py_str = rffi.cast(PyStringObject, py_obj) + py_str.c_size = len(space.str_w(w_obj)) + py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO) + def string_realize(space, ref): state = space.fromcache(State) ref = rffi.cast(PyStringObject, ref) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 15 00:45:54 2010 @@ -280,6 +280,7 @@ def init_unicodeobject(space): make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, + attach=type_attach, dealloc=type_dealloc) def c_type_descr__call__(space, w_type, __args__): @@ -383,7 +384,7 @@ Py_DecRef(space, pto) -def allocate_type_obj(space, py_obj, w_type): +def type_attach(space, py_obj, w_type): """ Allocates a PyTypeObject from a w_type which must be a PyPy type. """ from pypy.module.cpyext.object import PyObject_dealloc, PyObject_Del Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Thu Apr 15 00:45:54 2010 @@ -21,6 +21,7 @@ def init_unicodeobject(space): make_typedescr(space.w_unicode.instancetypedef, basestruct=PyUnicodeObject.TO, + attach=unicode_attach, dealloc=unicode_dealloc) # Buffer for the default encoding (used by PyUnicde_GetDefaultEncoding) @@ -32,6 +33,11 @@ Py_UNICODE = lltype.UniChar +def unicode_attach(space, py_obj, w_obj): + py_unicode = rffi.cast(PyUnicodeObject, py_obj) + py_unicode.c_size = len(space.unicode_w(w_obj)) + py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) + @cpython_api([PyObject], lltype.Void, external=False) def unicode_dealloc(space, obj): obj = rffi.cast(PyUnicodeObject, obj) From afa at codespeak.net Thu Apr 15 01:12:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Apr 2010 01:12:15 +0200 (CEST) Subject: [pypy-svn] r73751 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414231215.48E00282B9C@codespeak.net> Author: afa Date: Thu Apr 15 01:12:13 2010 New Revision: 73751 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: no more special case for tp_dealloc Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 15 01:12:13 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.baseobjspace import W_Root, Wrappable, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, \ +from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ PyObject, ADDR,\ Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State @@ -47,7 +47,10 @@ tp_dealloc.api_func.functype, tp_dealloc.api_func.get_wrapper(space)) else: - return null_dealloc # XXX PyObject_dealloc? + from pypy.module.cpyext.typeobject import subtype_dealloc + return llhelper( + subtype_dealloc.api_func.functype, + subtype_dealloc.api_func.get_wrapper(space)) if tp_alloc: def allocate(self, space, w_type): @@ -81,7 +84,12 @@ typedescr_cache[typedef] = CpyTypedescr() -make_typedescr(None) + at bootstrap_function +def init_pyobject(space): + from pypy.module.cpyext.object import PyObject_dealloc + make_typedescr(None) + make_typedescr(space.w_object.instancetypedef, + dealloc=PyObject_dealloc) @specialize.memo() def _get_typedescr_1(typedef): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 15 01:12:13 2010 @@ -386,27 +386,16 @@ def type_attach(space, py_obj, w_type): """ Allocates a PyTypeObject from a w_type which must be a PyPy type. """ - from pypy.module.cpyext.object import PyObject_dealloc, PyObject_Del + from pypy.module.cpyext.object import PyObject_Del assert isinstance(w_type, W_TypeObject) pto = rffi.cast(PyTypeObjectPtr, py_obj) + typedescr = get_typedescr(w_type.instancetypedef) + # dealloc - if space.is_w(w_type, space.w_object): - pto.c_tp_dealloc = llhelper(PyObject_dealloc.api_func.functype, - PyObject_dealloc.api_func.get_wrapper(space)) - elif space.is_w(w_type, space.w_type): - pto.c_tp_dealloc = llhelper(type_dealloc.api_func.functype, - type_dealloc.api_func.get_wrapper(space)) - #pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) - elif space.is_w(w_type, space.w_str): - pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) - elif space.is_w(w_type, space.w_unicode): - pto.c_tp_dealloc = get_typedescr(w_type.instancetypedef).get_dealloc(space) - else: - pto.c_tp_dealloc = llhelper(subtype_dealloc.api_func.functype, - subtype_dealloc.api_func.get_wrapper(space)) + pto.c_tp_dealloc = typedescr.get_dealloc(space) # buffer protocol if space.is_w(w_type, space.w_str): setup_string_buffer_procs(space, pto) From afa at codespeak.net Thu Apr 15 01:14:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Apr 2010 01:14:13 +0200 (CEST) Subject: [pypy-svn] r73752 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414231413.C8C03282B9C@codespeak.net> Author: afa Date: Thu Apr 15 01:14:12 2010 New Revision: 73752 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Add comment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 15 01:14:12 2010 @@ -87,9 +87,11 @@ @bootstrap_function def init_pyobject(space): from pypy.module.cpyext.object import PyObject_dealloc - make_typedescr(None) + # typedescr for the 'object' type make_typedescr(space.w_object.instancetypedef, dealloc=PyObject_dealloc) + # almost all types, which should better inherit from object. + make_typedescr(None) @specialize.memo() def _get_typedescr_1(typedef): From afa at codespeak.net Thu Apr 15 01:25:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Apr 2010 01:25:01 +0200 (CEST) Subject: [pypy-svn] r73753 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414232501.21805282B9C@codespeak.net> Author: afa Date: Thu Apr 15 01:24:59 2010 New Revision: 73753 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: More typedescr, less code. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 15 01:24:59 2010 @@ -277,7 +277,7 @@ PyMember_SetOne(space, w_self, self.member, w_value) @bootstrap_function -def init_unicodeobject(space): +def init_typeobject(space): make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, attach=type_attach, @@ -420,14 +420,11 @@ ref = make_ref(space, bases_w[0]) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) PyPyType_Ready(space, pto, w_type) - if space.is_w(w_type, space.w_object): - pto.c_tp_basicsize = rffi.sizeof(PyObject.TO) - elif space.is_w(w_type, space.w_type): - pto.c_tp_basicsize = rffi.sizeof(PyTypeObject) - elif space.is_w(w_type, space.w_str): - pto.c_tp_basicsize = rffi.sizeof(get_typedescr(w_type.instancetypedef).basestruct) - elif pto.c_tp_base: + + if pto.c_tp_base: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize + else: + pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) # will be filled later on with the correct value # may not be 0 From afa at codespeak.net Thu Apr 15 01:29:12 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 15 Apr 2010 01:29:12 +0200 (CEST) Subject: [pypy-svn] r73754 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100414232912.9DD9C282B9C@codespeak.net> Author: afa Date: Thu Apr 15 01:29:11 2010 New Revision: 73754 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: This 'if' does not seem necessary Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Thu Apr 15 01:29:11 2010 @@ -414,12 +414,11 @@ assert len(bases_w) <= 1 pto.c_tp_base = lltype.nullptr(PyTypeObject) pto.c_tp_bases = lltype.nullptr(PyObject.TO) - if not space.is_w(w_type, space.w_type) and not \ - space.is_w(w_type, space.w_object): - if bases_w: - ref = make_ref(space, bases_w[0]) - pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) - PyPyType_Ready(space, pto, w_type) + + if bases_w: + ref = make_ref(space, bases_w[0]) + pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) + PyPyType_Ready(space, pto, w_type) if pto.c_tp_base: pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize From agaynor at codespeak.net Thu Apr 15 02:55:20 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Thu, 15 Apr 2010 02:55:20 +0200 (CEST) Subject: [pypy-svn] r73755 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20100415005520.3C0E5282B9C@codespeak.net> Author: agaynor Date: Thu Apr 15 02:55:18 2010 New Revision: 73755 Modified: pypy/trunk/pypy/objspace/std/formatting.py pypy/trunk/pypy/objspace/std/test/test_stringformat.py Log: Fixed an issue with string formatting on invalid unicode data. Modified: pypy/trunk/pypy/objspace/std/formatting.py ============================================================================== --- pypy/trunk/pypy/objspace/std/formatting.py (original) +++ pypy/trunk/pypy/objspace/std/formatting.py Thu Apr 15 02:55:18 2010 @@ -488,7 +488,18 @@ result = formatter.format() except NeedUnicodeFormattingError: # fall through to the unicode case - fmt = unicode(fmt) + try: + fmt = unicode(fmt) + except UnicodeDecodeError, e: + raise OperationError(space.w_UnicodeDecodeError, + space.newtuple([ + space.wrap(e.encoding), + space.wrap(e.object), + space.wrap(e.start), + space.wrap(e.end), + space.wrap(e.reason), + ]) + ) else: return space.wrap(result) else: Modified: pypy/trunk/pypy/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_stringformat.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_stringformat.py Thu Apr 15 02:55:18 2010 @@ -1,3 +1,4 @@ +# coding: utf-8 class AppTestStringObjectWithDict: @@ -170,6 +171,9 @@ raises(TypeError, '%c'.__mod__, ("bla",)) raises(TypeError, '%c'.__mod__, ("",)) raises(TypeError, '%c'.__mod__, (['c'],)) + + def test_broken_unicode(self): + raises(UnicodeDecodeError, 'N?zov: %s'.__mod__, u'Jerry') class AppTestWidthPrec: def test_width(self): From arigo at codespeak.net Thu Apr 15 09:44:07 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 09:44:07 +0200 (CEST) Subject: [pypy-svn] r73756 - pypy/trunk/pypy/module/_file Message-ID: <20100415074407.5CBDA282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 09:44:05 2010 New Revision: 73756 Modified: pypy/trunk/pypy/module/_file/interp_file.py pypy/trunk/pypy/module/_file/interp_stream.py Log: Cleanup: there was already self.w_name, so adding self.name is just confusing. Modified: pypy/trunk/pypy/module/_file/interp_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_file.py (original) +++ pypy/trunk/pypy/module/_file/interp_file.py Thu Apr 15 09:44:05 2010 @@ -24,9 +24,10 @@ # Default values until the file is successfully opened stream = None - name = "" + w_name = None mode = "" - encoding = None + softspace= 0 # Required according to file object docs + encoding = None # This is not used internally by file objects fd = -1 def __init__(self, space): @@ -38,11 +39,8 @@ self.clear_all_weakrefs() self.direct_close() - def fdopenstream(self, stream, fd, mode, w_name): + def fdopenstream(self, stream, fd, mode): self.fd = fd - self.w_name = w_name - self.softspace = 0 # Required according to file object docs - self.encoding = None # This is not used internally by file objects self.mode = mode self.stream = stream if stream.flushable(): @@ -82,12 +80,12 @@ def direct___init__(self, w_name, mode='r', buffering=-1): name = self.space.str_w(w_name) - self.name = name self.direct_close() + self.w_name = w_name self.check_mode_ok(mode) stream = streamio.open_file_as_stream(name, mode, buffering) fd = stream.try_to_find_file_descriptor() - self.fdopenstream(stream, fd, mode, w_name) + self.fdopenstream(stream, fd, mode) def direct___enter__(self): if self.stream is None: @@ -103,9 +101,10 @@ def direct_fdopen(self, fd, mode='r', buffering=-1): self.direct_close() + self.w_name = self.space.wrap('') self.check_mode_ok(mode) stream = streamio.fdopen_as_stream(fd, mode, buffering) - self.fdopenstream(stream, fd, mode, self.space.wrap('')) + self.fdopenstream(stream, fd, mode) def direct_close(self): space = self.space @@ -240,7 +239,7 @@ try: self.direct_fdopen(fd, mode, buffering) except StreamErrors, e: - raise wrap_streamerror(self.space, e, self.name) + raise wrap_streamerror(self.space, e, self.w_name) _exposed_method_names = [] @@ -276,7 +275,7 @@ try: result = self.direct_%(name)s(%(callsig)s) except StreamErrors, e: - raise wrap_streamerror(space, e, self.name) + raise wrap_streamerror(space, e, self.w_name) finally: self.unlock() return %(wrapresult)s @@ -388,16 +387,19 @@ head = "closed" else: head = "open" - if self.space.is_true(self.space.isinstance(self.w_name, + w_name = self.w_name + if w_name is None: + w_name = self.space.wrap('?') + if self.space.is_true(self.space.isinstance(w_name, self.space.w_str)): info = "%s file '%s', mode '%s'" % ( head, - self.space.str_w(self.w_name), + self.space.str_w(w_name), self.mode) else: info = "%s file %s, mode '%s'" % ( head, - self.space.str_w(self.space.repr(self.w_name)), + self.space.str_w(self.space.repr(w_name)), self.mode) return self.getrepr(self.space, info) file__repr__.unwrap_spec = ['self'] Modified: pypy/trunk/pypy/module/_file/interp_stream.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_stream.py (original) +++ pypy/trunk/pypy/module/_file/interp_stream.py Thu Apr 15 09:44:05 2010 @@ -10,16 +10,16 @@ import os -def wrap_streamerror(space, e, filename): +def wrap_streamerror(space, e, w_filename=None): if isinstance(e, streamio.StreamError): return OperationError(space.w_ValueError, space.wrap(e.message)) elif isinstance(e, OSError): - return wrap_oserror_as_ioerror(space, e, filename) + return wrap_oserror_as_ioerror(space, e, w_filename) else: return OperationError(space.w_IOError, space.w_None) -def wrap_oserror_as_ioerror(space, e, filename): +def wrap_oserror_as_ioerror(space, e, w_filename=None): assert isinstance(e, OSError) errno = e.errno try: @@ -29,7 +29,7 @@ w_error = space.call_function(space.w_IOError, space.wrap(errno), space.wrap(msg), - space.wrap(filename)) + w_filename) return OperationError(space.w_IOError, w_error) From arigo at codespeak.net Thu Apr 15 09:59:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 09:59:08 +0200 (CEST) Subject: [pypy-svn] r73757 - pypy/trunk/pypy/objspace/std Message-ID: <20100415075908.B815A282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 09:59:07 2010 New Revision: 73757 Modified: pypy/trunk/pypy/objspace/std/formatting.py pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/objspace/std/unicodetype.py Log: Fix non-RPython code: can't read the attributes of a UnicodeDecodeError. Modified: pypy/trunk/pypy/objspace/std/formatting.py ============================================================================== --- pypy/trunk/pypy/objspace/std/formatting.py (original) +++ pypy/trunk/pypy/objspace/std/formatting.py Thu Apr 15 09:59:07 2010 @@ -329,7 +329,8 @@ length = len(r) if do_unicode and isinstance(r, str): # convert string to unicode explicitely here - r = unicode(r) + from pypy.objspace.std.unicodetype import plain_str2unicode + r = plain_str2unicode(self.space, r) prec = self.prec if prec == -1 and self.width == 0: # fast path @@ -488,18 +489,8 @@ result = formatter.format() except NeedUnicodeFormattingError: # fall through to the unicode case - try: - fmt = unicode(fmt) - except UnicodeDecodeError, e: - raise OperationError(space.w_UnicodeDecodeError, - space.newtuple([ - space.wrap(e.encoding), - space.wrap(e.object), - space.wrap(e.start), - space.wrap(e.end), - space.wrap(e.reason), - ]) - ) + from pypy.objspace.std.unicodetype import plain_str2unicode + fmt = plain_str2unicode(space, fmt) else: return space.wrap(result) else: Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Thu Apr 15 09:59:07 2010 @@ -39,25 +39,10 @@ W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)] del i -def _decode_ascii(space, s): - try: - return s.decode("ascii") - except UnicodeDecodeError: - for i in range(len(s)): - if ord(s[i]) > 127: - raise OperationError( - space.w_UnicodeDecodeError, - space.newtuple([ - space.wrap('ascii'), - space.wrap(s), - space.wrap(i), - space.wrap(i+1), - space.wrap("ordinal not in range(128)")])) - assert False, "unreachable" - def unicode_w__String(space, w_self): # XXX should this use the default encoding? - return _decode_ascii(space, w_self._value) + from pypy.objspace.std.unicodetype import plain_str2unicode + return plain_str2unicode(space, w_self._value) def _is_generic(space, w_self, fun): v = w_self._value Modified: pypy/trunk/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodetype.py (original) +++ pypy/trunk/pypy/objspace/std/unicodetype.py Thu Apr 15 09:59:07 2010 @@ -13,6 +13,22 @@ return wrapunicode(space, uni) return W_UnicodeObject(uni) +def plain_str2unicode(space, s): + try: + return unicode(s) + except UnicodeDecodeError: + for i in range(len(s)): + if ord(s[i]) > 127: + raise OperationError( + space.w_UnicodeDecodeError, + space.newtuple([ + space.wrap('ascii'), + space.wrap(s), + space.wrap(i), + space.wrap(i+1), + space.wrap("ordinal not in range(128)")])) + assert False, "unreachable" + unicode_capitalize = SMM('capitalize', 1, doc='S.capitalize() -> unicode\n\nReturn a' From arigo at codespeak.net Thu Apr 15 10:45:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 10:45:26 +0200 (CEST) Subject: [pypy-svn] r73758 - pypy/trunk/pypy/module/_file Message-ID: <20100415084526.2D56D282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 10:45:24 2010 New Revision: 73758 Modified: pypy/trunk/pypy/module/_file/interp_file.py Log: Fix for external callers of fdopenstream(), e.g. module/imp and module/bz2. Modified: pypy/trunk/pypy/module/_file/interp_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_file.py (original) +++ pypy/trunk/pypy/module/_file/interp_file.py Thu Apr 15 10:45:24 2010 @@ -39,9 +39,11 @@ self.clear_all_weakrefs() self.direct_close() - def fdopenstream(self, stream, fd, mode): + def fdopenstream(self, stream, fd, mode, w_name=None): self.fd = fd self.mode = mode + if w_name is not None: + self.w_name = w_name self.stream = stream if stream.flushable(): getopenstreams(self.space)[stream] = None From arigo at codespeak.net Thu Apr 15 11:22:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 11:22:45 +0200 (CEST) Subject: [pypy-svn] r73760 - in pypy/trunk/pypy/module: _file bz2 Message-ID: <20100415092245.2F20F282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 11:22:43 2010 New Revision: 73760 Modified: pypy/trunk/pypy/module/_file/interp_file.py pypy/trunk/pypy/module/bz2/interp_bz2.py Log: Fix translation (hopefully). Modified: pypy/trunk/pypy/module/_file/interp_file.py ============================================================================== --- pypy/trunk/pypy/module/_file/interp_file.py (original) +++ pypy/trunk/pypy/module/_file/interp_file.py Thu Apr 15 11:22:43 2010 @@ -389,22 +389,22 @@ head = "closed" else: head = "open" + info = "%s file %s, mode '%s'" % ( + head, + self.getdisplayname(), + self.mode) + return self.getrepr(self.space, info) + file__repr__.unwrap_spec = ['self'] + + def getdisplayname(self): w_name = self.w_name if w_name is None: - w_name = self.space.wrap('?') - if self.space.is_true(self.space.isinstance(w_name, - self.space.w_str)): - info = "%s file '%s', mode '%s'" % ( - head, - self.space.str_w(w_name), - self.mode) + return '?' + elif self.space.is_true(self.space.isinstance(w_name, + self.space.w_str)): + return "'%s'" % self.space.str_w(w_name) else: - info = "%s file %s, mode '%s'" % ( - head, - self.space.str_w(self.space.repr(w_name)), - self.mode) - return self.getrepr(self.space, info) - file__repr__.unwrap_spec = ['self'] + return self.space.str_w(self.space.repr(w_name)) def file_readinto(self, w_rwbuffer): """readinto() -> Undocumented. Don't use this; it may go away.""" Modified: pypy/trunk/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/trunk/pypy/module/bz2/interp_bz2.py (original) +++ pypy/trunk/pypy/module/bz2/interp_bz2.py Thu Apr 15 11:22:43 2010 @@ -222,7 +222,11 @@ head = "closed" else: head = "open" - info = "%s bz2.BZ2File '%s', mode '%s'" % (head, self.name, self.mode) + w_name = self.w_name + if w_name is None: + w_name = self.space.wrap('?') + info = "%s bz2.BZ2File %s, mode '%s'" % (head, self.getdisplayname(), + self.mode) return self.getrepr(self.space, info) file_bz2__repr__.unwrap_spec = ['self'] From arigo at codespeak.net Thu Apr 15 11:45:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 11:45:22 +0200 (CEST) Subject: [pypy-svn] r73761 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100415094522.E13D6282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 11:45:21 2010 New Revision: 73761 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Log: Start to write down some tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py Thu Apr 15 11:45:21 2010 @@ -229,6 +229,7 @@ def make_one_bytecode(self, graph_key, portal, called_from=None): maker = BytecodeMaker(self, graph_key, portal) if not hasattr(maker.bytecode, 'code'): + maker.generate() maker.assemble() self.counter += 1 if not self.counter % 500: @@ -389,8 +390,9 @@ assert codewriter.is_candidate(graph) self.graph = graph - def assemble(self): - """Assemble the opcodes for self.bytecode.""" + def generate(self): + """Generate in self.assembler the assembler corresponding to + this graph.""" self.assembler = [] self.constants = [] self.positions = {} @@ -403,6 +405,8 @@ exc_handler = self.pending_exception_handlers.pop() self.make_exception_handler(exc_handler) + def assemble(self): + """Assemble the opcodes for self.bytecode.""" labelpos = {} code = assemble(labelpos, self.codewriter.metainterp_sd, self.assembler) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Thu Apr 15 11:45:21 2010 @@ -3,7 +3,7 @@ from pypy.jit.metainterp import support, typesystem from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.history import ConstInt -from pypy.jit.metainterp.codewriter import CodeWriter +from pypy.jit.metainterp.codewriter import CodeWriter, BytecodeMaker from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.translator.translator import graphof from pypy.rpython.lltypesystem.rbuiltin import ll_instantiate @@ -174,6 +174,104 @@ rtyper = self.metainterp_sd.cpu.rtyper return graphof(rtyper.annotator.translator, func) + def encoding_test(self, func, args, ...): + graphs = self.make_graphs(func, args) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = graphs + cw._start(self.metainterp_sd, None) + maker = BytecodeMaker(cw, (graphs[0], None), False) + maker.generate() + ... + + def test_bytecodemaker_generate_simple(self): + def f(n): + return n + 10 + self.encoding_test(f, [5], """ + [%i0] + int_add %i0 $10 %i0 + int_return %i0 + """) + + def test_bytecodemaker_generate_regs(self): + def f(a, b, c): + return (((a + 10) * 2) + (b - c)) * a + self.encoding_test(f, [5, 6, 7], """ + [%i0, %i1, %i2] + int_add %i0 $10 %i3 + int_mul %i3 $2 %i3 + int_sub %i1 %i2 %i1 + int_add %i3 %i1 %i1 + int_mul %i1 %i0 %i0 + int_return %i0 + """) + + def test_bytecodemaker_generate_loop(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + L0: + int_gt %i0 $0 %i2 + goto_if_not %i2 L1 + int_add %i1 %i0 %i1 + int_sub %i0 $1 %i0 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_swap(self): + def f(a, b): + while a > 0: + a, b = b, b+a + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + L0: + int_gt %i0 $0 %i2 + goto_if_not %i2 L1 + int_add %i1 %i0 %i0 + int_swap %i0 %i1 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_cycle(self): + def f(a, b, c): + while a > 0: + a, b, c = b, c, a + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1, %i2] + L0: + int_gt %i0 $0 %i3 + goto_if_not %i3 L1 + int_swap_cycle [%i0, %i2, %i1] + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_same_as(self): + def f(a, b, c): + while a > 0: + b = c + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1, %i2] + L0: + int_gt %i0 $0 %i3 + goto_if_not %i3 L1 + int_same_as %i2 %i1 + goto L0 + L1: + int_return %i1 + """) + def test_basic(self): def f(n): return n + 10 From arigo at codespeak.net Thu Apr 15 11:49:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 11:49:41 +0200 (CEST) Subject: [pypy-svn] r73762 - pypy/branch/blackhole-improvement/pypy/jit/metainterp/test Message-ID: <20100415094941.A8BC6282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 11:49:40 2010 New Revision: 73762 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Log: Tweak the tests. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Thu Apr 15 11:49:40 2010 @@ -214,8 +214,7 @@ self.encoding_test(f, [5, 6], """ [%i0, %i1] L0: - int_gt %i0 $0 %i2 - goto_if_not %i2 L1 + goto_if_not_int_gt %i0 $0 L1 int_add %i1 %i0 %i1 int_sub %i0 $1 %i0 goto L0 @@ -231,8 +230,7 @@ self.encoding_test(f, [5, 6], """ [%i0, %i1] L0: - int_gt %i0 $0 %i2 - goto_if_not %i2 L1 + goto_if_not_int_gt %i0 $0 L1 int_add %i1 %i0 %i0 int_swap %i0 %i1 goto L0 @@ -248,15 +246,14 @@ self.encoding_test(f, [5, 6], """ [%i0, %i1, %i2] L0: - int_gt %i0 $0 %i3 - goto_if_not %i3 L1 + goto_if_not_int_gt %i0 $0 L1 int_swap_cycle [%i0, %i2, %i1] goto L0 L1: int_return %i1 """) - def test_bytecodemaker_generate_same_as(self): + def test_bytecodemaker_generate_same_as_var(self): def f(a, b, c): while a > 0: b = c @@ -264,14 +261,28 @@ self.encoding_test(f, [5, 6], """ [%i0, %i1, %i2] L0: - int_gt %i0 $0 %i3 - goto_if_not %i3 L1 + goto_if_not_int_gt %i0 $0 L1 int_same_as %i2 %i1 goto L0 L1: int_return %i1 """) + def test_bytecodemaker_generate_same_as_const(self): + def f(a, b, c): + while a > 0: + b = -17 + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1, %i2] + L0: + goto_if_not_int_gt %i0 $0 L1 + int_same_as $-17 %i1 + goto L0 + L1: + int_return %i1 + """) + def test_basic(self): def f(n): return n + 10 From arigo at codespeak.net Thu Apr 15 13:17:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 13:17:46 +0200 (CEST) Subject: [pypy-svn] r73763 - in pypy/trunk/pypy/objspace/flow: . test Message-ID: <20100415111746.0A324282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 13:17:44 2010 New Revision: 73763 Modified: pypy/trunk/pypy/objspace/flow/objspace.py pypy/trunk/pypy/objspace/flow/test/test_objspace.py Log: Test and fix. Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Thu Apr 15 13:17:44 2010 @@ -492,6 +492,7 @@ float: [ValueError], chr: [ValueError], unichr: [ValueError], + unicode: [UnicodeDecodeError], # specifying IndexError, and KeyError beyond Exception, # allows the annotator to be more precise, see test_reraiseAnything/KeyError in # the annotator tests Modified: pypy/trunk/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/test/test_objspace.py Thu Apr 15 13:17:44 2010 @@ -1,7 +1,7 @@ import new import py from pypy.objspace.flow.model import Constant, Block, Link, Variable, traverse -from pypy.objspace.flow.model import flatten, mkentrymap +from pypy.objspace.flow.model import flatten, mkentrymap, c_last_exception from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph from pypy.objspace.flow.objspace import FlowObjSpace @@ -728,7 +728,18 @@ return unicode("1234") graph = self.codetest(myfunc) assert graph.startblock.exits[0].target is graph.returnblock - + + def test_unicode(self): + def myfunc(n): + try: + return unicode(chr(n)) + except UnicodeDecodeError: + return None + graph = self.codetest(myfunc) + simplify_graph(graph) + assert graph.startblock.exitswitch == c_last_exception + assert graph.startblock.exits[0].target is graph.returnblock + assert graph.startblock.exits[1].target is graph.returnblock def test_getitem(self): def f(c, x): From arigo at codespeak.net Thu Apr 15 15:27:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 15 Apr 2010 15:27:22 +0200 (CEST) Subject: [pypy-svn] r73764 - in pypy/branch/blackhole-improvement/pypy: jit/metainterp jit/metainterp/test tool/algo Message-ID: <20100415132722.9C99B282B9C@codespeak.net> Author: arigo Date: Thu Apr 15 15:27:20 2010 New Revision: 73764 Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter2.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter2.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Log: For now, start from scratch a codewriter2. The coloring tests pass. Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter2.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter2.py Thu Apr 15 15:27:20 2010 @@ -0,0 +1,77 @@ +import sys +from pypy.tool.algo.color import DependencyGraph +from pypy.tool.algo.unionfind import UnionFind +from pypy.objspace.flow.model import Variable + + +class BytecodeMaker(object): + DEBUG_REGALLOC = False + + def __init__(self, graph): + self.graph = graph + + def generate(self): + self.register_allocation() + + def register_allocation(self): + dg = DependencyGraph() + # + pendingblocks = list(self.graph.iterblocks()) + for block in pendingblocks: + # Compute die_at = {Variable: index_of_operation_with_last_usage} + die_at = dict.fromkeys(block.inputargs, 0) + for i, op in enumerate(block.operations): + for v in op.args: + if isinstance(v, Variable): + die_at[v] = i + if op.result is not None: + die_at[op.result] = i + die_at[block.exitswitch] = sys.maxint + for link in block.exits: + for v in link.args: + die_at[v] = sys.maxint + # Add the variables of this block to the dependency graph + for i, v in enumerate(block.inputargs): + dg.add_node(v) + for j in range(i): + dg.add_edge(block.inputargs[j], v) + livevars = set(block.inputargs) + die_at = [(value, key) for (key, value) in die_at.items()] + die_at.sort() + die_at.append((sys.maxint,)) + die_index = 0 + for i, op in enumerate(block.operations): + while die_at[die_index][0] == i: + livevars.remove(die_at[die_index][1]) + die_index += 1 + if op.result is not None: + livevars.add(op.result) + dg.add_node(op.result) + for v in livevars: + dg.add_edge(v, op.result) + # + uf = UnionFind() + while pendingblocks: + block = pendingblocks.pop() + # Aggressively try to coalesce each source variable with its target + for link in block.exits: + for i, v in enumerate(link.args): + if isinstance(v, Variable): + w = link.target.inputargs[i] + v0 = uf.find_rep(v) + w0 = uf.find_rep(w) + if v0 not in dg.neighbours[w0]: + _, rep, _ = uf.union(v0, w0) + assert rep is v0 + dg.coalesce(w0, v0) + # + self._coloring = dg.find_node_coloring() + self._unionfind = uf + if self.DEBUG_REGALLOC: + for block in self.graph.iterblocks(): + print block + for v in block.getvariables(): + print '\t', v, '\t', self.getcolor(v) + + def getcolor(self, v): + return self._coloring[self._unionfind.find_rep(v)] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Thu Apr 15 15:27:20 2010 @@ -174,115 +174,6 @@ rtyper = self.metainterp_sd.cpu.rtyper return graphof(rtyper.annotator.translator, func) - def encoding_test(self, func, args, ...): - graphs = self.make_graphs(func, args) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = graphs - cw._start(self.metainterp_sd, None) - maker = BytecodeMaker(cw, (graphs[0], None), False) - maker.generate() - ... - - def test_bytecodemaker_generate_simple(self): - def f(n): - return n + 10 - self.encoding_test(f, [5], """ - [%i0] - int_add %i0 $10 %i0 - int_return %i0 - """) - - def test_bytecodemaker_generate_regs(self): - def f(a, b, c): - return (((a + 10) * 2) + (b - c)) * a - self.encoding_test(f, [5, 6, 7], """ - [%i0, %i1, %i2] - int_add %i0 $10 %i3 - int_mul %i3 $2 %i3 - int_sub %i1 %i2 %i1 - int_add %i3 %i1 %i1 - int_mul %i1 %i0 %i0 - int_return %i0 - """) - - def test_bytecodemaker_generate_loop(self): - def f(a, b): - while a > 0: - b += a - a -= 1 - return b - self.encoding_test(f, [5, 6], """ - [%i0, %i1] - L0: - goto_if_not_int_gt %i0 $0 L1 - int_add %i1 %i0 %i1 - int_sub %i0 $1 %i0 - goto L0 - L1: - int_return %i1 - """) - - def test_bytecodemaker_generate_swap(self): - def f(a, b): - while a > 0: - a, b = b, b+a - return b - self.encoding_test(f, [5, 6], """ - [%i0, %i1] - L0: - goto_if_not_int_gt %i0 $0 L1 - int_add %i1 %i0 %i0 - int_swap %i0 %i1 - goto L0 - L1: - int_return %i1 - """) - - def test_bytecodemaker_generate_cycle(self): - def f(a, b, c): - while a > 0: - a, b, c = b, c, a - return b - self.encoding_test(f, [5, 6], """ - [%i0, %i1, %i2] - L0: - goto_if_not_int_gt %i0 $0 L1 - int_swap_cycle [%i0, %i2, %i1] - goto L0 - L1: - int_return %i1 - """) - - def test_bytecodemaker_generate_same_as_var(self): - def f(a, b, c): - while a > 0: - b = c - return b - self.encoding_test(f, [5, 6], """ - [%i0, %i1, %i2] - L0: - goto_if_not_int_gt %i0 $0 L1 - int_same_as %i2 %i1 - goto L0 - L1: - int_return %i1 - """) - - def test_bytecodemaker_generate_same_as_const(self): - def f(a, b, c): - while a > 0: - b = -17 - return b - self.encoding_test(f, [5, 6], """ - [%i0, %i1, %i2] - L0: - goto_if_not_int_gt %i0 $0 L1 - int_same_as $-17 %i1 - goto L0 - L1: - int_return %i1 - """) - def test_basic(self): def f(n): return n + 10 Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter2.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter2.py Thu Apr 15 15:27:20 2010 @@ -0,0 +1,165 @@ +from pypy.jit.metainterp import support +from pypy.jit.metainterp.codewriter2 import BytecodeMaker + + +class TestBytecodeMaker: + + def make_graphs(self, func, values, type_system='lltype'): + self.rtyper = support.annotate(func, values, type_system=type_system) + return self.rtyper.annotator.translator.graphs + + def coloring_test(self, func, args, expected_lambda): + graphs = self.make_graphs(func, args) + maker = BytecodeMaker(graphs[0]) + maker.register_allocation() + expected = expected_lambda(graphs[0].startblock) + for v, num in expected.items(): + print '\t%s\tcolor %d\texpected %d' % (v, maker.getcolor(v), num) + for v, num in expected.items(): + assert maker.getcolor(v) == num + + def encoding_test(self, func, args, expected): + graphs = self.make_graphs(func, args) + maker = BytecodeMaker(graphs[0]) + maker.generate() + asm = maker.format_assembler() + expected = str(py.code.Source(expected)).strip() + assert asm == expected + '\n' + + def test_coloring_simple(self): + def f(n): + return n + 10 + self.coloring_test(f, [5], + lambda startblock: + {startblock.inputargs[0]: 0, + startblock.operations[0].result: 0, + startblock.exits[0].target.inputargs[0]: 0}) + + def test_coloring_bigblock(self): + def f(a, b, c): + return (((a + 10) * 2) + (b - c)) * a + self.coloring_test(f, [5, 6, 7], + lambda startblock: + {startblock.inputargs[0]: 0, + startblock.inputargs[1]: 1, + startblock.inputargs[2]: 2, + startblock.operations[0].result: 3, + startblock.operations[1].result: 3, + startblock.operations[2].result: 1, + startblock.operations[3].result: 1, + startblock.operations[4].result: 0, + startblock.exits[0].target.inputargs[0]: 0}) + + def test_bytecodemaker_generate_simple(self): + def f(n): + return n + 10 + self.encoding_test(f, [5], """ + [%i0] + int_add %i0, $10, %i0 + int_return %i0 + """) + + def test_bytecodemaker_generate_bigblock(self): + def f(a, b, c): + return (((a + 10) * 2) + (b - c)) * a + self.encoding_test(f, [5, 6, 7], """ + [%i0, %i1, %i2] + int_add %i0, $10, %i3 + int_mul %i3, $2, %i3 + int_sub %i1, %i2, %i1 + int_add %i3, %i1, %i1 + int_mul %i1, %i0, %i0 + int_return %i0 + """) + + def test_bytecodemaker_generate_loop(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + L0: + goto_if_not_int_gt %i0, $0, L1 + int_add %i1, %i0, %i1 + int_sub %i0, $1, %i0 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_swap(self): + def f(a, b): + while a > 0: + a, b = b, b+a + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + L0: + goto_if_not_int_gt %i0, $0, L1 + int_add %i1, %i0, %i0 + int_swap %i0, %i1 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_cycle(self): + def f(a, b, c): + while a > 0: + a, b, c = b, c, a + return b + self.encoding_test(f, [5, 6, 7], """ + [%i0, %i1, %i2] + L0: + goto_if_not_int_gt %i0, $0, L1 + int_swap_cycle [%i0, %i2, %i1] + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_same_as_var(self): + def f(a, b, c): + while a > 0: + b = c + return b + self.encoding_test(f, [5, 6, 7], """ + [%i0, %i1, %i2] + L0: + goto_if_not_int_gt %i0, $0, L1 + int_same_as %i2, %i1 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_same_as_const(self): + def f(a, b): + while a > 0: + b = -17 + return b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + L0: + goto_if_not_int_gt %i0, $0, L1 + int_same_as $-17, %i1 + goto L0 + L1: + int_return %i1 + """) + + def test_bytecodemaker_generate_return_const(self): + def f(a, b): + if a > b: + b = -17 + return 1 + b + self.encoding_test(f, [5, 6], """ + [%i0, %i1] + goto_if_not_int_gt %i0, %i1, L1 + int_return $-16 + L1: + int_add $1, %i1, %i0 + int_return %i0 + """) Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Thu Apr 15 15:27:20 2010 @@ -2,21 +2,33 @@ class DependencyGraph(object): def __init__(self): - self.nodes = [] + self._all_nodes = [] self.neighbours = {} def add_node(self, v): assert v not in self.neighbours, "duplicate vertex %r" % (v,) - self.nodes.append(v) + self._all_nodes.append(v) self.neighbours[v] = set() def add_edge(self, v1, v2): self.neighbours[v1].add(v2) self.neighbours[v2].add(v1) + def coalesce(self, vold, vnew): + """Remove vold from the graph, and attach all its edges to vnew.""" + for n in self.neighbours.pop(vold): + self.neighbours[n].remove(vold) + self.neighbours[n].add(vnew) + self.neighbours[vnew].add(n) + # we should remove vold from self._all_nodes, but it's too costly + # so we rely on getnodes() to filter self._all_nodes. + + def getnodes(self): + return [v for v in self._all_nodes if v in self.neighbours] + def lexicographic_order(self): """Enumerate a lexicographic breath-first ordering of the nodes.""" - sigma = [self.nodes[:]] + sigma = [self.getnodes()[::-1]] while sigma: v = sigma[0].pop() yield v From fijal at codespeak.net Thu Apr 15 21:47:32 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 21:47:32 +0200 (CEST) Subject: [pypy-svn] r73788 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100415194732.C23B6282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 21:47:31 2010 New Revision: 73788 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: finish calls, fun Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Thu Apr 15 21:47:31 2010 @@ -44,7 +44,6 @@ assert ret.val == 100 def test_function(self): - py.test.skip("in progress") code = compile(''' func: # arg comes in r0 LOAD 1 => r1 Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Thu Apr 15 21:47:31 2010 @@ -109,7 +109,16 @@ self.code += [JUMP_IF_ABOVE, self.rint(arg0.strip()), self.rint(arg1.strip()), self.labels[label[1:]]] - #def compile_LOAD_FUNCTION(self, + def compile_LOAD_FUNCTION(self, args): + name, res = args.split("=>") + no, code = self.functions[name.strip()] + self.code += [LOAD_FUNCTION, no, self.rint(res.strip())] + + def compile_CALL(self, args): + args, res = args.split("=>") + arg0, arg1 = args.strip().split(" ") + self.code += [CALL, self.rint(arg0.strip()), self.rint(arg1.strip()), + self.rint(res.strip())] def compile(strrepr): parser = Parser() @@ -138,6 +147,15 @@ def gt(self, other): return self.val > other.val +class Func(Object): + def __init__(self, code): + self.code = code + + def call(self, arg): + f = Frame(self.code) + f.registers[0] = arg + return f.interpret() + class Frame(object): def __init__(self, code): self.code = code @@ -166,6 +184,16 @@ i = tgt else: i += 4 + elif opcode == LOAD_FUNCTION: + f = self.code.functions[ord(code[i + 1])] + self.registers[ord(code[i + 2])] = Func(f) + i += 3 + elif opcode == CALL: + f = self.registers[ord(code[i + 1])] + arg = self.registers[ord(code[i + 2])] + assert isinstance(f, Func) + self.registers[ord(code[i + 3])] = f.call(arg) + i += 4 else: raise Exception("unimplemented opcode %s" % opcodes[opcode]) From fijal at codespeak.net Thu Apr 15 21:51:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 21:51:27 +0200 (CEST) Subject: [pypy-svn] r73789 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100415195127.53A86282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 21:51:25 2010 New Revision: 73789 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: Function combination Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Thu Apr 15 21:51:25 2010 @@ -57,3 +57,24 @@ ''') ret = interpret(code) assert ret.val == 1 + 1 + + def test_function_combination(self): + code = compile(''' + inner: + LOAD 2 => r1 + ADD r1 r0 => r0 + RETURN r0 + outer: + LOAD 1 => r1 + ADD r1 r0 => r2 + RETURN r2 + main: + LOAD_FUNCTION inner => r0 + LOAD_FUNCTION outer => r1 + ADD r1 r0 => r2 + LOAD 1 => r3 + CALL r2 r3 => r4 + RETURN r4 + ''') + ret = interpret(code) + assert ret.val == 1 + 1 + 2 Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Thu Apr 15 21:51:25 2010 @@ -156,6 +156,17 @@ f.registers[0] = arg return f.interpret() + def add(self, other): + return CombinedFunc(self, other) + +class CombinedFunc(Func): + def __init__(self, outer, inner): + self.outer = outer + self.inner = inner + + def call(self, arg): + return self.outer.call(self.inner.call(arg)) + class Frame(object): def __init__(self, code): self.code = code From fijal at codespeak.net Thu Apr 15 22:01:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:01:24 +0200 (CEST) Subject: [pypy-svn] r73790 - in pypy/trunk/pypy/jit/tl: . test Message-ID: <20100415200124.AEF06282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:01:23 2010 New Revision: 73790 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: implement print Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Thu Apr 15 22:01:23 2010 @@ -78,3 +78,21 @@ ''') ret = interpret(code) assert ret.val == 1 + 1 + 2 + + def test_print(self): + import sys + from StringIO import StringIO + + code = compile(''' + main: + LOAD 0 => r1 + PRINT r1 + RETURN r1 + ''') + s = StringIO() + sys.stdout = s + try: + interpret(code) + assert s.getvalue() == '0\n' + finally: + sys.stdout = sys.__stdout__ Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Thu Apr 15 22:01:23 2010 @@ -1,5 +1,8 @@ -""" Interpreter for a tiny interpreter with frame introspection. Supports +""" +Run tinyframe.py [int value reg0] [int value reg1] ... + +Interpreter for a tiny interpreter with frame introspection. Supports integer values and function values. The machine is register based with untyped registers. @@ -24,16 +27,19 @@ function argument always comes in r0 """ +from pypy.rlib.streamio import open_file_as_stream + opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', 'RETURN', 'JUMP', 'JUMP_IF_ABOVE'] for i, opcode in enumerate(opcodes): globals()[opcode] = i class Code(object): - def __init__(self, code, regno, functions): + def __init__(self, code, regno, functions, name): self.code = code self.regno = regno self.functions = functions + self.name = name class Parser(object): @@ -65,14 +71,14 @@ functions = [code for i, code in sorted(self.functions.values())] assert self.name == 'main' return Code("".join([chr(i) for i in self.code]), self.maxregno + 1, - functions) + functions, self.name) def finish_currect_code(self): if self.name is None: assert not self.code return code = Code("".join([chr(i) for i in self.code]), self.maxregno + 1, - []) + [], self.name) self.functions[self.name] = (len(self.functions), code) self.name = None self.labels = {} @@ -137,6 +143,9 @@ def gt(self, other): raise NotImplementedError("abstract base class") + def repr(self): + raise NotImplementedError("abstract base class") + class Int(Object): def __init__(self, val): self.val = val @@ -147,6 +156,9 @@ def gt(self, other): return self.val > other.val + def repr(self): + return str(self.val) + class Func(Object): def __init__(self, code): self.code = code @@ -159,6 +171,9 @@ def add(self, other): return CombinedFunc(self, other) + def repr(self): + return "" % self.code.name + class CombinedFunc(Func): def __init__(self, outer, inner): self.outer = outer @@ -167,6 +182,9 @@ def call(self, arg): return self.outer.call(self.inner.call(arg)) + def repr(self): + return "" % (self.outer.repr(), self.inner.repr()) + class Frame(object): def __init__(self, code): self.code = code @@ -205,8 +223,30 @@ assert isinstance(f, Func) self.registers[ord(code[i + 3])] = f.call(arg) i += 4 + elif opcode == PRINT: + arg = self.registers[ord(code[i + 1])] + print arg.repr() + i += 2 else: raise Exception("unimplemented opcode %s" % opcodes[opcode]) def interpret(code): return Frame(code).interpret() + +def main(fname, argv): + f = open_file_as_stream(fname, "r") + input = f.read() + f.close() + code = compile(input) + mainframe = Frame(code) + for i in range(len(argv)): + mainframe.registers[i] = Int(int(argv[i])) + print "Result:", mainframe.interpret().repr() + +if __name__ == '__main__': + import sys + if len(sys.argv < 2): + print __doc__ + sys.exit(1) + fname = sys.argv[1] + main(fname, sys.argv[2:]) From fijal at codespeak.net Thu Apr 15 22:03:52 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:03:52 +0200 (CEST) Subject: [pypy-svn] r73791 - pypy/trunk/pypy/jit/tl/test Message-ID: <20100415200352.C291B282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:03:51 2010 New Revision: 73791 Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Log: improve the test Modified: pypy/trunk/pypy/jit/tl/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_tinyframe.py Thu Apr 15 22:03:51 2010 @@ -84,15 +84,26 @@ from StringIO import StringIO code = compile(''' + name: + RETURN r0 main: LOAD 0 => r1 PRINT r1 + LOAD_FUNCTION name => r1 + PRINT r1 + ADD r1 r1 => r2 + PRINT r2 RETURN r1 ''') s = StringIO() sys.stdout = s try: interpret(code) - assert s.getvalue() == '0\n' finally: sys.stdout = sys.__stdout__ + lines = s.getvalue().splitlines() + assert lines == [ + '0', + '', + '()>', + ] From fijal at codespeak.net Thu Apr 15 22:19:31 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:19:31 +0200 (CEST) Subject: [pypy-svn] r73792 - pypy/trunk/pypy/jit/tl Message-ID: <20100415201931.011E5282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:19:29 2010 New Revision: 73792 Added: pypy/trunk/pypy/jit/tl/support.py (contents, props changed) pypy/trunk/pypy/jit/tl/targettinyframe.py (contents, props changed) Modified: pypy/trunk/pypy/jit/tl/tinyframe.py Log: Translation Added: pypy/trunk/pypy/jit/tl/support.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/support.py Thu Apr 15 22:19:29 2010 @@ -0,0 +1,22 @@ + +def partition(array, left, right): + last_item = array[right] + pivot = last_item[0] + storeindex = left + for i in range(left, right): + if array[i][0] <= pivot: + array[i], array[storeindex] = array[storeindex], array[i] + storeindex += 1 + # Move pivot to its final place + array[storeindex], array[right] = last_item, array[storeindex] + return storeindex + +def quicksort(array, left, right): + # sort array[left:right+1] (i.e. bounds included) + if right > left: + pivotnewindex = partition(array, left, right) + quicksort(array, left, pivotnewindex - 1) + quicksort(array, pivotnewindex + 1, right) + +def sort(array): + quicksort(array, 0, len(array) - 1) Added: pypy/trunk/pypy/jit/tl/targettinyframe.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/targettinyframe.py Thu Apr 15 22:19:29 2010 @@ -0,0 +1,11 @@ + +from pypy.jit.tl.tinyframe import main + +def entry_point(argv): + main(argv[0], argv[1:]) + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None Modified: pypy/trunk/pypy/jit/tl/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe.py Thu Apr 15 22:19:29 2010 @@ -28,9 +28,12 @@ """ from pypy.rlib.streamio import open_file_as_stream +from pypy.jit.tl.support import sort +from pypy.rlib.unroll import unrolling_iterable opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', 'RETURN', 'JUMP', 'JUMP_IF_ABOVE'] +unrolling_opcodes = unrolling_iterable(opcodes) for i, opcode in enumerate(opcodes): globals()[opcode] = i @@ -54,8 +57,9 @@ for line in lines: comment = line.find('#') if comment != -1: + assert comment >= 0 line = line[:comment] - line = line.strip() + line = line.strip(" ") if not line: continue if line.endswith(':'): @@ -66,9 +70,16 @@ if line.startswith('@'): self.labels[line[1:]] = len(self.code) continue - opcode, args = line.split(" ", 1) - getattr(self, 'compile_' + opcode)(args) - functions = [code for i, code in sorted(self.functions.values())] + firstspace = line.find(" ") + assert firstspace >= 0 + opcode = line[:firstspace] + args = line[firstspace + 1:] + for name in unrolling_opcodes: + if opcode == name: + getattr(self, 'compile_' + name)(args) + values = self.functions.values() + sort(values) + functions = [code for i, code in values] assert self.name == 'main' return Code("".join([chr(i) for i in self.code]), self.maxregno + 1, functions, self.name) @@ -92,39 +103,50 @@ return no def compile_ADD(self, args): - args, result = args.split("=>") - arg0, arg1 = args.strip().split(" ") + args, result = args.split("=") + result = result[1:] + arg0, arg1 = args.strip(" ").split(" ") self.code += [ADD, self.rint(arg0), self.rint(arg1), - self.rint(result.strip())] + self.rint(result.strip(" "))] def compile_LOAD(self, args): - arg0, result = args.split("=>") - arg0 = arg0.strip() - self.code += [LOAD, int(arg0), self.rint(result.strip())] + arg0, result = args.split("=") + result = result[1:] + arg0 = arg0.strip(" ") + self.code += [LOAD, int(arg0), self.rint(result.strip(" "))] def compile_PRINT(self, args): - arg = self.rint(args.strip()) + arg = self.rint(args.strip(" ")) self.code += [PRINT, arg] def compile_RETURN(self, args): - arg = self.rint(args.strip()) + arg = self.rint(args.strip(" ")) self.code += [RETURN, arg] def compile_JUMP_IF_ABOVE(self, args): arg0, arg1, label = args.split(" ") - self.code += [JUMP_IF_ABOVE, self.rint(arg0.strip()), - self.rint(arg1.strip()), self.labels[label[1:]]] + self.code += [JUMP_IF_ABOVE, self.rint(arg0.strip(" ")), + self.rint(arg1.strip(" ")), self.labels[label[1:]]] def compile_LOAD_FUNCTION(self, args): - name, res = args.split("=>") - no, code = self.functions[name.strip()] - self.code += [LOAD_FUNCTION, no, self.rint(res.strip())] + name, res = args.split("=") + res = res[1:] + no, code = self.functions[name.strip(" ")] + self.code += [LOAD_FUNCTION, no, self.rint(res.strip(" "))] def compile_CALL(self, args): - args, res = args.split("=>") - arg0, arg1 = args.strip().split(" ") - self.code += [CALL, self.rint(arg0.strip()), self.rint(arg1.strip()), - self.rint(res.strip())] + args, res = args.split("=") + res = res[1:] + arg0, arg1 = args.strip(" ").split(" ") + self.code += [CALL, self.rint(arg0.strip(" ")), + self.rint(arg1.strip(" ")), + self.rint(res.strip(" "))] + + def compile_INTROSPECT(self, args): + raise NotImplementedError + + def compile_JUMP(self, args): + raise NotImplementedError def compile(strrepr): parser = Parser() @@ -235,7 +257,7 @@ def main(fname, argv): f = open_file_as_stream(fname, "r") - input = f.read() + input = f.readall() f.close() code = compile(input) mainframe = Frame(code) From fijal at codespeak.net Thu Apr 15 22:21:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:21:55 +0200 (CEST) Subject: [pypy-svn] r73793 - in pypy/trunk/pypy/jit/tl: . test tinyframe tinyframe/test Message-ID: <20100415202155.BCA2F282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:21:53 2010 New Revision: 73793 Added: pypy/trunk/pypy/jit/tl/tinyframe/ pypy/trunk/pypy/jit/tl/tinyframe/__init__.py (contents, props changed) pypy/trunk/pypy/jit/tl/tinyframe/support.py - copied unchanged from r73792, pypy/trunk/pypy/jit/tl/support.py pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py - copied unchanged from r73792, pypy/trunk/pypy/jit/tl/targettinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/test/ pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py - copied, changed from r73791, pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py - copied, changed from r73792, pypy/trunk/pypy/jit/tl/tinyframe.py Removed: pypy/trunk/pypy/jit/tl/support.py pypy/trunk/pypy/jit/tl/targettinyframe.py pypy/trunk/pypy/jit/tl/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe.py Log: Move the tinyframe interpreter to it's own directory Added: pypy/trunk/pypy/jit/tl/tinyframe/__init__.py ============================================================================== Copied: pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py (from r73791, pypy/trunk/pypy/jit/tl/test/test_tinyframe.py) ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py Thu Apr 15 22:21:53 2010 @@ -1,6 +1,6 @@ import py -from pypy.jit.tl.tinyframe import * +from pypy.jit.tl.tinyframe.tinyframe import * class TestCompile(object): def test_simple(self): Copied: pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py (from r73792, pypy/trunk/pypy/jit/tl/tinyframe.py) ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Thu Apr 15 22:21:53 2010 @@ -28,7 +28,7 @@ """ from pypy.rlib.streamio import open_file_as_stream -from pypy.jit.tl.support import sort +from pypy.jit.tl.tinyframe.support import sort from pypy.rlib.unroll import unrolling_iterable opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', From fijal at codespeak.net Thu Apr 15 22:25:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:25:22 +0200 (CEST) Subject: [pypy-svn] r73794 - in pypy/trunk/pypy/jit/tl/tinyframe: . examples Message-ID: <20100415202522.2F1B1282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:25:20 2010 New Revision: 73794 Added: pypy/trunk/pypy/jit/tl/tinyframe/examples/ pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf Modified: pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Log: Example and small fixes Added: pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf Thu Apr 15 22:25:20 2010 @@ -0,0 +1,6 @@ +main: +LOAD 0 => r1 +LOAD 2 => r2 +ADD r1 r2 => r3 +PRINT r3 +RETURN r3 \ No newline at end of file Modified: pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py Thu Apr 15 22:25:20 2010 @@ -1,5 +1,5 @@ -from pypy.jit.tl.tinyframe import main +from pypy.jit.tl.tinyframe.tinyframe import main def entry_point(argv): main(argv[0], argv[1:]) Modified: pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Thu Apr 15 22:25:20 2010 @@ -267,7 +267,7 @@ if __name__ == '__main__': import sys - if len(sys.argv < 2): + if len(sys.argv) < 2: print __doc__ sys.exit(1) fname = sys.argv[1] From fijal at codespeak.net Thu Apr 15 22:27:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:27:17 +0200 (CEST) Subject: [pypy-svn] r73795 - in pypy/trunk/pypy/jit/tl/tinyframe: . examples Message-ID: <20100415202717.5E41D282B9D@codespeak.net> Author: fijal Date: Thu Apr 15 22:27:15 2010 New Revision: 73795 Modified: pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Log: cosmetic fixes Modified: pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/examples/simple.tf Thu Apr 15 22:27:15 2010 @@ -3,4 +3,4 @@ LOAD 2 => r2 ADD r1 r2 => r3 PRINT r3 -RETURN r3 \ No newline at end of file +RETURN r3 Modified: pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py Thu Apr 15 22:27:15 2010 @@ -2,7 +2,7 @@ from pypy.jit.tl.tinyframe.tinyframe import main def entry_point(argv): - main(argv[0], argv[1:]) + main(argv[1], argv[2:]) return 0 # _____ Define and setup target ___ Modified: pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Thu Apr 15 22:27:15 2010 @@ -263,7 +263,8 @@ mainframe = Frame(code) for i in range(len(argv)): mainframe.registers[i] = Int(int(argv[i])) - print "Result:", mainframe.interpret().repr() + res = mainframe.interpret() + print "Result:", res.repr() if __name__ == '__main__': import sys From fijal at codespeak.net Thu Apr 15 22:29:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:29:55 +0200 (CEST) Subject: [pypy-svn] r73796 - pypy/trunk/pypy/jit/tl/tinyframe/examples Message-ID: <20100415202955.7FA37282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:29:54 2010 New Revision: 73796 Added: pypy/trunk/pypy/jit/tl/tinyframe/examples/loop.tf Log: the simplest loop example Added: pypy/trunk/pypy/jit/tl/tinyframe/examples/loop.tf ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/tinyframe/examples/loop.tf Thu Apr 15 22:29:54 2010 @@ -0,0 +1,7 @@ +main: +LOAD 0 => r1 +LOAD 1 => r2 + at add +ADD r2 r1 => r1 +JUMP_IF_ABOVE r0 r1 @add +RETURN r1 From fijal at codespeak.net Thu Apr 15 22:53:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 15 Apr 2010 22:53:45 +0200 (CEST) Subject: [pypy-svn] r73797 - pypy/trunk/pypy/jit/tl/tinyframe Message-ID: <20100415205345.BB64D282B9C@codespeak.net> Author: fijal Date: Thu Apr 15 22:53:44 2010 New Revision: 73797 Modified: pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Log: jit work Modified: pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/targettinyframe.py Thu Apr 15 22:53:44 2010 @@ -1,5 +1,9 @@ from pypy.jit.tl.tinyframe.tinyframe import main +from pypy.jit.metainterp.policy import JitPolicy + +def jitpolicy(driver): + return JitPolicy() def entry_point(argv): main(argv[1], argv[2:]) Modified: pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Thu Apr 15 22:53:44 2010 @@ -30,6 +30,7 @@ from pypy.rlib.streamio import open_file_as_stream from pypy.jit.tl.tinyframe.support import sort from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.jit import JitDriver, hint opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', 'RETURN', 'JUMP', 'JUMP_IF_ABOVE'] @@ -186,8 +187,7 @@ self.code = code def call(self, arg): - f = Frame(self.code) - f.registers[0] = arg + f = Frame(self.code, arg) return f.interpret() def add(self, other): @@ -207,15 +207,23 @@ def repr(self): return "" % (self.outer.repr(), self.inner.repr()) +driver = JitDriver(greens = ['code', 'i'], reds = ['self'], + virtualizables = ['self']) + class Frame(object): - def __init__(self, code): + _virtualizable2_ = ['registers[*]', 'code'] + + def __init__(self, code, arg=None): + self = hint(self, access_directly=True, fresh_virtualizable=True) self.code = code - self.registers = [None] * code.regno + self.registers = [None] * code.regno + self.registers[0] = arg def interpret(self): i = 0 code = self.code.code while True: + driver.jit_merge_point(self=self, code=code, i=i) opcode = ord(code[i]) if opcode == LOAD: self.registers[ord(code[i + 2])] = Int(ord(code[i + 1])) @@ -233,6 +241,7 @@ tgt = ord(code[i + 3]) if arg0.gt(arg1): i = tgt + driver.can_enter_jit(code=code, i=tgt, self=self) else: i += 4 elif opcode == LOAD_FUNCTION: From fijal at codespeak.net Fri Apr 16 01:32:07 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 01:32:07 +0200 (CEST) Subject: [pypy-svn] r73800 - in pypy/trunk/pypy/jit/tl/tinyframe: . test Message-ID: <20100415233207.9C925282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 01:32:05 2010 New Revision: 73800 Modified: pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Log: Implement INTROSPECT Modified: pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/test/test_tinyframe.py Fri Apr 16 01:32:05 2010 @@ -107,3 +107,14 @@ '', '()>', ] + + def test_introspect(self): + code = compile(''' + main: + LOAD 100 => r0 + LOAD 0 => r1 + INTROSPECT r1 => r2 + RETURN r0 + ''') + res = interpret(code) + assert res.val == 100 Modified: pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py (original) +++ pypy/trunk/pypy/jit/tl/tinyframe/tinyframe.py Fri Apr 16 01:32:05 2010 @@ -30,7 +30,7 @@ from pypy.rlib.streamio import open_file_as_stream from pypy.jit.tl.tinyframe.support import sort from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.jit import JitDriver, hint, dont_look_inside opcodes = ['ADD', 'INTROSPECT', 'PRINT', 'CALL', 'LOAD', 'LOAD_FUNCTION', 'RETURN', 'JUMP', 'JUMP_IF_ABOVE'] @@ -144,7 +144,10 @@ self.rint(res.strip(" "))] def compile_INTROSPECT(self, args): - raise NotImplementedError + arg, res = args.split("=") + res = res[1:] + self.code += [INTROSPECT, self.rint(arg.strip(" ")), + self.rint(res.strip(" "))] def compile_JUMP(self, args): raise NotImplementedError @@ -258,9 +261,18 @@ arg = self.registers[ord(code[i + 1])] print arg.repr() i += 2 + elif opcode == INTROSPECT: + self.introspect(ord(code[i + 1]), ord(code[i + 2])) + i += 3 else: raise Exception("unimplemented opcode %s" % opcodes[opcode]) + @dont_look_inside + def introspect(self, rarg, rresult): + source = self.registers[rarg] + assert isinstance(source, Int) + self.registers[rresult] = self.registers[source.val] + def interpret(code): return Frame(code).interpret() From fijal at codespeak.net Fri Apr 16 05:19:19 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 05:19:19 +0200 (CEST) Subject: [pypy-svn] r73802 - pypy/trunk/pypy/module/sys Message-ID: <20100416031919.8958A282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 05:19:18 2010 New Revision: 73802 Modified: pypy/trunk/pypy/module/sys/app.py Log: update copyright, reindent docstring Modified: pypy/trunk/pypy/module/sys/app.py ============================================================================== --- pypy/trunk/pypy/module/sys/app.py (original) +++ pypy/trunk/pypy/module/sys/app.py Fri Apr 16 05:19:18 2010 @@ -28,7 +28,7 @@ def getfilesystemencoding(): """Return the encoding used to convert Unicode filenames in -operating system filenames. + operating system filenames. """ if sys.platform == "win32": encoding = "mbcs" @@ -43,7 +43,7 @@ return None copyright_str = """ -Copyright 2003-2009 PyPy development team. +Copyright 2003-2010 PyPy development team. All rights reserved. For further information see http://www.codespeak.net/pypy. From afa at codespeak.net Fri Apr 16 10:40:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 10:40:38 +0200 (CEST) Subject: [pypy-svn] r73803 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416084038.DB025282B9C@codespeak.net> Author: afa Date: Fri Apr 16 10:40:36 2010 New Revision: 73803 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Change the logic to bootstrap basic types (object, type, tuple) - insert null references in the w2r table - then create references - fix cycles - update the w2r table with complete objects This removes most of special cases in make_ref. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 16 10:40:36 2010 @@ -435,27 +435,52 @@ return func def bootstrap_types(space): - from pypy.module.cpyext.pyobject import make_ref - from pypy.module.cpyext.typeobject import PyTypeObjectPtr, PyPyType_Ready, \ - inherit_slots + from pypy.module.cpyext.pyobject import make_ref, create_ref, track_reference + from pypy.module.cpyext.typeobject import PyTypeObjectPtr for func in BOOTSTRAP_FUNCTIONS: func(space) - # bootstrap this damn cycle - type_pto = make_ref(space, space.w_type) - type_pto = rffi.cast(PyTypeObjectPtr, type_pto) - - object_pto = make_ref(space, space.w_object) - object_pto = rffi.cast(PyTypeObjectPtr, object_pto) - type_pto.c_tp_base = object_pto - type_pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_type)) - object_pto.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_type)) - PyPyType_Ready(space, object_pto, space.w_object) - PyPyType_Ready(space, type_pto, space.w_type) - type_pto.c_tp_bases = make_ref(space, space.newtuple([space.w_object])) - object_pto.c_tp_bases = make_ref(space, space.newtuple([])) - inherit_slots(space, type_pto, space.w_object) + # some types are difficult to create because of cycles. + # - object.ob_type = type + # - type.ob_type = type + # - tuple.ob_type = type + # - type.tp_base = object + # - tuple.tp_base = object + # - type.tp_bases is a tuple + # - object.tp_bases is a tuple + # - tuple.tp_bases is a tuple + + # insert null placeholders to please make_ref() + state = space.fromcache(State) + state.py_objects_w2r[space.w_type] = lltype.nullptr(PyObject.TO) + state.py_objects_w2r[space.w_object] = lltype.nullptr(PyObject.TO) + state.py_objects_w2r[space.w_tuple] = lltype.nullptr(PyObject.TO) + + # create the objects + py_type = create_ref(space, space.w_type) + py_object = create_ref(space, space.w_object) + py_tuple = create_ref(space, space.w_tuple) + + # form cycles + pto_type = rffi.cast(PyTypeObjectPtr, py_type) + py_type.c_ob_type = pto_type + py_object.c_ob_type = pto_type + py_tuple.c_ob_type = pto_type + + pto_object = rffi.cast(PyTypeObjectPtr, py_object) + pto_type.c_tp_base = pto_object + pto_tuple.c_tp_base = pto_object + + pto_tuple = rffi.cast(PyTypeObjectPtr, py_tuple) + pto_type.c_tp_bases.c_ob_type = pto_tuple + pto_object.c_tp_bases.c_ob_type = pto_tuple + pto_tuple.c_tp_bases.c_ob_type = pto_tuple + + # Restore the mapping + track_reference(space, py_type, space.w_type) + track_reference(space, py_object, space.w_object) + track_reference(space, py_tuple, space.w_tuple) #_____________________________________________________ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 10:40:36 2010 @@ -62,8 +62,7 @@ obj = lltype.malloc(tp_basestruct, flavor='raw', zero=True) pyobj = rffi.cast(PyObject, obj) pyobj.c_ob_refcnt = 1 - if w_type is not space.w_type: - pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) return pyobj if tp_attach: @@ -134,20 +133,7 @@ w_type = space.type(w_obj) typedescr = get_typedescr(w_obj.typedef) - if space.is_w(w_type, space.w_type) or space.is_w(w_type, - space.gettypeobject(W_PyCTypeObject.typedef)): - py_obj = typedescr.allocate(space, w_type) - - # put the type object early into the dict - # to support dependency cycles like object/type - state = space.fromcache(State) - state.py_objects_w2r[w_obj] = py_obj - py_obj.c_ob_type = rffi.cast( - PyTypeObjectPtr, make_ref(space, w_type, - steal=not space.is_w(w_type, space.w_type))) - - typedescr.attach(space, py_obj, w_obj) - elif isinstance(w_type, W_PyCTypeObject): + if isinstance(w_type, W_PyCTypeObject): lifeline = w_obj.get_pyolifeline() if lifeline is not None: # make old PyObject ready for use in C code py_obj = lifeline.pyo @@ -171,23 +157,31 @@ typedescr.attach(space, py_obj, w_obj) return py_obj +def track_reference(space, py_obj, w_obj, borrowed=False): + # XXX looks like a PyObject_GC_TRACK + ptr = rffi.cast(ADDR, py_obj) + if DEBUG_REFCOUNT: + debug_refcount("MAKREF", py_obj, w_obj) + state = space.fromcache(State) + state.py_objects_w2r[w_obj] = py_obj + state.py_objects_r2w[ptr] = w_obj + if borrowed and ptr not in state.borrowed_objects: + state.borrowed_objects[ptr] = None + def make_ref(space, w_obj, borrowed=False, steal=False, items=0): if w_obj is None: return lltype.nullptr(PyObject.TO) assert isinstance(w_obj, W_Root) state = space.fromcache(State) - py_obj = state.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) - if not py_obj: + try: + py_obj = state.py_objects_w2r[w_obj] + except KeyError: assert not steal py_obj = create_ref(space, w_obj, items) - ptr = rffi.cast(ADDR, py_obj) - if DEBUG_REFCOUNT: - debug_refcount("MAKREF", py_obj, w_obj) - state.py_objects_w2r[w_obj] = py_obj - state.py_objects_r2w[ptr] = w_obj - if borrowed and ptr not in state.borrowed_objects: - state.borrowed_objects[ptr] = None - elif not steal: + track_reference(space, py_obj, w_obj, borrowed=borrowed) + return py_obj + + if not steal: if borrowed: py_obj_addr = rffi.cast(ADDR, py_obj) if py_obj_addr not in state.borrowed_objects: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 10:40:36 2010 @@ -457,8 +457,7 @@ assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 pto.c_tp_flags |= Py_TPFLAGS_READYING base = pto.c_tp_base - if not base and not (w_obj is not None and - space.is_w(w_obj, space.w_object)): + if not base and not space.is_w(w_obj, space.w_object): base_pyo = make_ref(space, space.w_object, steal=True) base = pto.c_tp_base = rffi.cast(PyTypeObjectPtr, base_pyo) else: @@ -467,8 +466,7 @@ PyPyType_Ready(space, base, None) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type - if not pto.c_tp_bases and not (space.is_w(w_obj, space.w_object) - or space.is_w(w_obj, space.w_type)): + if not pto.c_tp_bases: if not base: bases = space.newtuple([]) else: From afa at codespeak.net Fri Apr 16 10:46:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 10:46:32 +0200 (CEST) Subject: [pypy-svn] r73804 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416084632.DC265282B9C@codespeak.net> Author: afa Date: Fri Apr 16 10:46:30 2010 New Revision: 73804 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: oops Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Fri Apr 16 10:46:30 2010 @@ -470,9 +470,9 @@ pto_object = rffi.cast(PyTypeObjectPtr, py_object) pto_type.c_tp_base = pto_object + pto_tuple = rffi.cast(PyTypeObjectPtr, py_tuple) pto_tuple.c_tp_base = pto_object - pto_tuple = rffi.cast(PyTypeObjectPtr, py_tuple) pto_type.c_tp_bases.c_ob_type = pto_tuple pto_object.c_tp_bases.c_ob_type = pto_tuple pto_tuple.c_tp_bases.c_ob_type = pto_tuple From afa at codespeak.net Fri Apr 16 11:03:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 11:03:53 +0200 (CEST) Subject: [pypy-svn] r73805 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416090353.557D1282B9C@codespeak.net> Author: afa Date: Fri Apr 16 11:03:51 2010 New Revision: 73805 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: from_ref() is now completely generic. Progress! Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 11:03:51 2010 @@ -7,7 +7,6 @@ Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject -from pypy.objspace.std.objectobject import W_ObjectObject from pypy.rlib.objectmodel import specialize, we_are_translated from pypy.rpython.annlowlevel import llhelper @@ -77,7 +76,9 @@ return tp_realize(space, ref) else: def realize(self, space, ref): - raise TypeError("cannot realize") + # For most types, a reference cannot exist without + # a real interpreter object + raise InvalidPointerException(str(ref)) if typedef: CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,) @@ -193,31 +194,24 @@ def from_ref(space, ref, recurse=False): - from pypy.module.cpyext.typeobject import PyPyType_Ready assert lltype.typeOf(ref) == PyObject if not ref: return None state = space.fromcache(State) ptr = rffi.cast(ADDR, ref) + try: - w_obj = state.py_objects_r2w[ptr] + return state.py_objects_r2w[ptr] except KeyError: if recurse: raise InvalidPointerException(str(ref)) - ref_type = rffi.cast(PyObject, ref.c_ob_type) - if ref != ref_type: - w_type = from_ref(space, ref_type, True) - assert isinstance(w_type, W_TypeObject) - if space.is_w(w_type, space.w_str): - return get_typedescr(w_type.instancetypedef).realize(space, ref) - elif space.is_w(w_type, space.w_type): - PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) - return from_ref(space, ref, True) - else: - raise InvalidPointerException(str(ref)) - else: - raise InvalidPointerException("This should never happen") - return w_obj + + # This reference is not yet a real interpreter object. + # Realize it. + ref_type = rffi.cast(PyObject, ref.c_ob_type) + w_type = from_ref(space, ref_type, True) + assert isinstance(w_type, W_TypeObject) + return get_typedescr(w_type.instancetypedef).realize(space, ref) # XXX Optimize these functions and put them into macro definitions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 11:03:51 2010 @@ -281,6 +281,7 @@ make_typedescr(space.w_type.instancetypedef, basestruct=PyTypeObject, attach=type_attach, + realize=type_realize, dealloc=type_dealloc) def c_type_descr__call__(space, w_type, __args__): @@ -449,6 +450,10 @@ finally: Py_DecRef(space, base_pyo) +def type_realize(space, ref): + PyPyType_Ready(space, rffi.cast(PyTypeObjectPtr, ref), None) + return from_ref(space, ref, True) + def PyPyType_Ready(space, pto, w_obj): try: pto.c_tp_dict = lltype.nullptr(PyObject.TO) # not supported From afa at codespeak.net Fri Apr 16 11:56:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 11:56:09 +0200 (CEST) Subject: [pypy-svn] r73806 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416095609.DB94C282B9C@codespeak.net> Author: afa Date: Fri Apr 16 11:55:51 2010 New Revision: 73806 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Improve debug message when refcount is negative Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 11:55:51 2010 @@ -130,7 +130,6 @@ def create_ref(space, w_obj, items=0): from pypy.module.cpyext.typeobject import W_PyCTypeObject, PyOLifeline - from pypy.module.cpyext.pycobject import W_PyCObject, PyCObject w_type = space.type(w_obj) typedescr = get_typedescr(w_obj.typedef) @@ -264,8 +263,10 @@ del state.borrow_mapping[ptr] else: if not we_are_translated() and obj.c_ob_refcnt < 0: - print >>sys.stderr, "Negative refcount for obj %s with type %s" % (obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) - assert False + message = "Negative refcount for obj %s with type %s" % ( + obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) + print >>sys.stderr, message + assert False, message @cpython_api([PyObject], lltype.Void) def Py_IncRef(space, obj): From afa at codespeak.net Fri Apr 16 13:13:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 13:13:45 +0200 (CEST) Subject: [pypy-svn] r73807 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416111345.6C2A7282B9C@codespeak.net> Author: afa Date: Fri Apr 16 13:13:29 2010 New Revision: 73807 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: More simplification, allocate() can now handle variable-sized objects. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 13:13:29 2010 @@ -52,16 +52,31 @@ subtype_dealloc.api_func.get_wrapper(space)) if tp_alloc: - def allocate(self, space, w_type): + def allocate(self, space, w_type, itemcount=0): return tp_alloc(space, w_type) else: - def allocate(self, space, w_type): + def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? # except that it's not related to any pypy object. - obj = lltype.malloc(tp_basestruct, flavor='raw', zero=True) - pyobj = rffi.cast(PyObject, obj) + + pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + # Don't increase refcount for non-heaptypes + if pytype: + flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) + if not flags & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, w_type) + + if pytype: + size = pytype.c_tp_basicsize + else: + size = rffi.sizeof(tp_basestruct) + if itemcount: + size += itemcount * pytype.c_tp_itemsize + buf = lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True) + pyobj = rffi.cast(PyObject, buf) pyobj.c_ob_refcnt = 1 - pyobj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + pyobj.c_ob_type = pytype return pyobj if tp_attach: @@ -140,20 +155,10 @@ assert py_obj.c_ob_refcnt == 0 Py_IncRef(space, py_obj) else: - w_type_pyo = make_ref(space, w_type) - pto = rffi.cast(PyTypeObjectPtr, w_type_pyo) - # Don't increase refcount for non-heaptypes - if not rffi.cast(lltype.Signed, pto.c_tp_flags) & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, w_type_pyo) - basicsize = pto.c_tp_basicsize + items * pto.c_tp_itemsize - py_obj_pad = lltype.malloc(rffi.VOIDP.TO, basicsize, - flavor="raw", zero=True) - py_obj = rffi.cast(PyObject, py_obj_pad) - py_obj.c_ob_refcnt = 1 - py_obj.c_ob_type = pto + py_obj = typedescr.allocate(space, w_type, itemcount=items) w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) else: - py_obj = get_typedescr(w_obj.typedef).allocate(space, w_type) + py_obj = typedescr.allocate(space, w_type, itemcount=items) typedescr.attach(space, py_obj, w_obj) return py_obj From afa at codespeak.net Fri Apr 16 15:05:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 15:05:53 +0200 (CEST) Subject: [pypy-svn] r73808 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416130553.A7184282B9C@codespeak.net> Author: afa Date: Fri Apr 16 15:04:46 2010 New Revision: 73808 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Move some blocks Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 15:04:46 2010 @@ -201,27 +201,6 @@ pto.c_tp_new = base_pto.c_tp_new -class W_PyCTypeObject(W_TypeObject): - def __init__(self, space, pto): - bases_w = space.fixedview(from_ref(space, pto.c_tp_bases)) - dict_w = {} - - add_operators(space, dict_w, pto) - convert_method_defs(space, dict_w, pto.c_tp_methods, pto) - convert_getset_defs(space, dict_w, pto.c_tp_getset, pto) - convert_member_defs(space, dict_w, pto.c_tp_members, pto) - - full_name = rffi.charp2str(pto.c_tp_name) - if '.' in full_name: - module_name, extension_name = rsplit(full_name, ".", 1) - dict_w["__module__"] = space.wrap(module_name) - else: - extension_name = full_name - - W_TypeObject.__init__(self, space, extension_name, - bases_w or [space.w_object], dict_w) - self.__flags__ = _CPYTYPE # mainly disables lookup optimizations - class __extend__(W_Root): __metaclass__ = extendabletype __slots__ = ("_pyolifeline", ) # hint for the annotator @@ -276,14 +255,6 @@ check_descr(space, w_self, self.pto) PyMember_SetOne(space, w_self, self.member, w_value) - at bootstrap_function -def init_typeobject(space): - make_typedescr(space.w_type.instancetypedef, - basestruct=PyTypeObject, - attach=type_attach, - realize=type_realize, - dealloc=type_dealloc) - def c_type_descr__call__(space, w_type, __args__): if isinstance(w_type, W_PyCTypeObject): pyo = make_ref(space, w_type) @@ -323,6 +294,40 @@ w_type.ready() return w_type +class W_PyCTypeObject(W_TypeObject): + def __init__(self, space, pto): + bases_w = space.fixedview(from_ref(space, pto.c_tp_bases)) + dict_w = {} + + add_operators(space, dict_w, pto) + convert_method_defs(space, dict_w, pto.c_tp_methods, pto) + convert_getset_defs(space, dict_w, pto.c_tp_getset, pto) + convert_member_defs(space, dict_w, pto.c_tp_members, pto) + + full_name = rffi.charp2str(pto.c_tp_name) + if '.' in full_name: + module_name, extension_name = rsplit(full_name, ".", 1) + dict_w["__module__"] = space.wrap(module_name) + else: + extension_name = full_name + + W_TypeObject.__init__(self, space, extension_name, + bases_w or [space.w_object], dict_w) + self.__flags__ = _CPYTYPE # mainly disables lookup optimizations + +W_PyCTypeObject.typedef = TypeDef( + 'C_type', W_TypeObject.typedef, + __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), + __new__ = interp2app(c_type_descr__new__), + ) + + at bootstrap_function +def init_typeobject(space): + make_typedescr(space.w_type.instancetypedef, + basestruct=PyTypeObject, + attach=type_attach, + realize=type_realize, + dealloc=type_dealloc) @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): @@ -508,12 +513,6 @@ w_obj.ready() return 1 -W_PyCTypeObject.typedef = TypeDef( - 'C_type', W_TypeObject.typedef, - __call__ = interp2app(c_type_descr__call__, unwrap_spec=[ObjSpace, W_Root, Arguments]), - __new__ = interp2app(c_type_descr__new__), - ) - @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): """Return true if a is a subtype of b. From afa at codespeak.net Fri Apr 16 15:08:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 15:08:43 +0200 (CEST) Subject: [pypy-svn] r73809 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416130843.86B45282B9C@codespeak.net> Author: afa Date: Fri Apr 16 15:08:41 2010 New Revision: 73809 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: No need to override allocate() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 15:08:41 2010 @@ -29,7 +29,6 @@ """ tp_basestruct = kw.get('basestruct', PyObject.TO) - tp_alloc = kw.get('alloc') tp_attach = kw.get('attach') tp_realize = kw.get('realize') tp_dealloc = kw.get('dealloc') @@ -51,33 +50,29 @@ subtype_dealloc.api_func.functype, subtype_dealloc.api_func.get_wrapper(space)) - if tp_alloc: - def allocate(self, space, w_type, itemcount=0): - return tp_alloc(space, w_type) - else: - def allocate(self, space, w_type, itemcount=0): - # similar to PyType_GenericAlloc? - # except that it's not related to any pypy object. - - pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) - # Don't increase refcount for non-heaptypes - if pytype: - flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) - if not flags & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, w_type) + def allocate(self, space, w_type, itemcount=0): + # similar to PyType_GenericAlloc? + # except that it's not related to any pypy object. + + pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + # Don't increase refcount for non-heaptypes + if pytype: + flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) + if not flags & Py_TPFLAGS_HEAPTYPE: + Py_DecRef(space, w_type) - if pytype: - size = pytype.c_tp_basicsize - else: - size = rffi.sizeof(tp_basestruct) - if itemcount: - size += itemcount * pytype.c_tp_itemsize - buf = lltype.malloc(rffi.VOIDP.TO, size, - flavor='raw', zero=True) - pyobj = rffi.cast(PyObject, buf) - pyobj.c_ob_refcnt = 1 - pyobj.c_ob_type = pytype - return pyobj + if pytype: + size = pytype.c_tp_basicsize + else: + size = rffi.sizeof(tp_basestruct) + if itemcount: + size += itemcount * pytype.c_tp_itemsize + buf = lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True) + pyobj = rffi.cast(PyObject, buf) + pyobj.c_ob_refcnt = 1 + pyobj.c_ob_type = pytype + return pyobj if tp_attach: def attach(self, space, pyobj, w_obj): From afa at codespeak.net Fri Apr 16 15:14:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 15:14:53 +0200 (CEST) Subject: [pypy-svn] r73810 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416131453.3F0FF282B9C@codespeak.net> Author: afa Date: Fri Apr 16 15:14:44 2010 New Revision: 73810 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Ensure that all keyword arguments to make_typedescr() are processed Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 15:14:44 2010 @@ -28,10 +28,11 @@ dealloc : a cpython_api(external=False), similar to PyObject_dealloc """ - tp_basestruct = kw.get('basestruct', PyObject.TO) - tp_attach = kw.get('attach') - tp_realize = kw.get('realize') - tp_dealloc = kw.get('dealloc') + tp_basestruct = kw.pop('basestruct', PyObject.TO) + tp_attach = kw.pop('attach', None) + tp_realize = kw.pop('realize', None) + tp_dealloc = kw.pop('dealloc', None) + assert not kw, "Extra arguments to make_typedescr" null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void)) From afa at codespeak.net Fri Apr 16 15:25:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 15:25:14 +0200 (CEST) Subject: [pypy-svn] r73811 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416132514.4075E282B9C@codespeak.net> Author: afa Date: Fri Apr 16 15:24:19 2010 New Revision: 73811 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Customize instances of W_PyCTypeObject subclasses: the new slot make_ref is defined on the metatype typedescr. Now make_ref() is completely generic Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 16 15:24:19 2010 @@ -1,6 +1,6 @@ import sys -from pypy.interpreter.baseobjspace import W_Root, Wrappable, SpaceCache +from pypy.interpreter.baseobjspace import W_Root, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ PyObject, ADDR,\ @@ -29,6 +29,7 @@ """ tp_basestruct = kw.pop('basestruct', PyObject.TO) + tp_make_ref = kw.pop('make_ref', None) tp_attach = kw.pop('attach', None) tp_realize = kw.pop('realize', None) tp_dealloc = kw.pop('dealloc', None) @@ -75,6 +76,18 @@ pyobj.c_ob_type = pytype return pyobj + # Specialized by meta-type + if tp_make_ref: + def make_ref(self, space, w_type, w_obj, itemcount=0): + return tp_make_ref(space, w_type, w_obj, itemcount=itemcount) + else: + def make_ref(self, space, w_type, w_obj, itemcount=0): + typedescr = get_typedescr(w_obj.typedef) + w_type = space.type(w_obj) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + typedescr.attach(space, py_obj, w_obj) + return py_obj + if tp_attach: def attach(self, space, pyobj, w_obj): tp_attach(space, pyobj, w_obj) @@ -140,23 +153,9 @@ print >>sys.stderr def create_ref(space, w_obj, items=0): - from pypy.module.cpyext.typeobject import W_PyCTypeObject, PyOLifeline w_type = space.type(w_obj) - typedescr = get_typedescr(w_obj.typedef) - - if isinstance(w_type, W_PyCTypeObject): - lifeline = w_obj.get_pyolifeline() - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - Py_IncRef(space, py_obj) - else: - py_obj = typedescr.allocate(space, w_type, itemcount=items) - w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) - else: - py_obj = typedescr.allocate(space, w_type, itemcount=items) - typedescr.attach(space, py_obj, w_obj) - return py_obj + metatypedescr = get_typedescr(w_type.typedef) + return metatypedescr.make_ref(space, w_type, w_obj, itemcount=items) def track_reference(space, py_obj, w_obj, borrowed=False): # XXX looks like a PyObject_GC_TRACK Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 15:24:19 2010 @@ -328,6 +328,12 @@ attach=type_attach, realize=type_realize, dealloc=type_dealloc) + make_typedescr(W_PyCTypeObject.typedef, + basestruct=PyTypeObject, + make_ref=pyctype_make_ref, + attach=type_attach, + realize=type_realize, + dealloc=type_dealloc) @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): @@ -345,6 +351,18 @@ # hopefully this does not clash with the memory model assumed in # extension modules +def pyctype_make_ref(space, w_type, w_obj, itemcount=0): + lifeline = w_obj.get_pyolifeline() + if lifeline is not None: # make old PyObject ready for use in C code + py_obj = lifeline.pyo + assert py_obj.c_ob_refcnt == 0 + Py_IncRef(space, py_obj) + else: + typedescr = get_typedescr(w_obj.typedef) + py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) + w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) + return py_obj + @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): From afa at codespeak.net Fri Apr 16 16:16:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 16:16:20 +0200 (CEST) Subject: [pypy-svn] r73812 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416141620.0C6A3282BDB@codespeak.net> Author: afa Date: Fri Apr 16 16:15:56 2010 New Revision: 73812 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: more cleanup, thanks to the better bootstrapping method Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 16:15:56 2010 @@ -485,7 +485,7 @@ assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 pto.c_tp_flags |= Py_TPFLAGS_READYING base = pto.c_tp_base - if not base and not space.is_w(w_obj, space.w_object): + if not base: base_pyo = make_ref(space, space.w_object, steal=True) base = pto.c_tp_base = rffi.cast(PyTypeObjectPtr, base_pyo) else: @@ -504,9 +504,8 @@ PyPyType_Register(space, pto) if base: inherit_special(space, pto, base) - if pto.c_tp_bases: # this is false while bootstrapping - for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) + for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): + inherit_slots(space, pto, w_base) # missing: # setting __doc__ if not defined and tp_doc defined # inheriting tp_as_* slots From afa at codespeak.net Fri Apr 16 16:21:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 16 Apr 2010 16:21:46 +0200 (CEST) Subject: [pypy-svn] r73813 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100416142146.4760C282BAD@codespeak.net> Author: afa Date: Fri Apr 16 16:21:36 2010 New Revision: 73813 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Remove a check: "flags & Py_TPFLAGS_HAVE_CLASS" is here for binary compatibility across cpython releases, so that a recent python can load modules compiled before the introduction of tp_new. This does not concern pypy. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Fri Apr 16 16:21:36 2010 @@ -192,14 +192,11 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way flags = rffi.cast(lltype.Signed, pto.c_tp_flags) - if True or flags & Py_TPFLAGS_HAVE_CLASS: # XXX i do not understand this check - base_object_pyo = make_ref(space, space.w_object, steal=True) - base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) - if base_pto != base_object_pto or \ - flags & Py_TPFLAGS_HEAPTYPE: - if not pto.c_tp_new: - pto.c_tp_new = base_pto.c_tp_new - + base_object_pyo = make_ref(space, space.w_object, steal=True) + base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) + if base_pto != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE: + if not pto.c_tp_new: + pto.c_tp_new = base_pto.c_tp_new class __extend__(W_Root): __metaclass__ = extendabletype From fijal at codespeak.net Fri Apr 16 19:52:45 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 19:52:45 +0200 (CEST) Subject: [pypy-svn] r73814 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100416175245.AD5D7282BAD@codespeak.net> Author: fijal Date: Fri Apr 16 19:52:44 2010 New Revision: 73814 Added: pypy/trunk/pypy/rlib/rlocale.py (contents, props changed) pypy/trunk/pypy/rlib/test/test_rlocale.py (contents, props changed) Log: Start adding rlocale, not used by anyone so far. Added: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 19:52:44 2010 @@ -0,0 +1,152 @@ + +""" The rpython-level part of locale module +""" + +import sys + +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.tool import rffi_platform as platform + +class LocaleError(Exception): + pass + +HAVE_LANGINFO = sys.platform != 'win32' +HAVE_LIBINTL = sys.platform != 'win32' + +class CConfig: + includes = ['locale.h', 'limits.h'] + if HAVE_LANGINFO: + includes += ['langinfo.h'] + if HAVE_LIBINTL: + includes += ['libintl.h'] + if sys.platform == 'win32': + includes += ['windows.h'] + _compilation_info_ = ExternalCompilationInfo( + includes=includes, + ) + HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') + lconv = platform.Struct("struct lconv", [ + # Numeric (non-monetary) information. + ("decimal_point", rffi.CCHARP), # Decimal point character. + ("thousands_sep", rffi.CCHARP), # Thousands separator. + + ## Each element is the number of digits in each group; + ## elements with higher indices are farther left. + ## An element with value CHAR_MAX means that no further grouping is done. + ## An element with value 0 means that the previous element is used + ## for all groups farther left. */ + ("grouping", rffi.CCHARP), + + ## Monetary information. + + ## First three chars are a currency symbol from ISO 4217. + ## Fourth char is the separator. Fifth char is '\0'. + ("int_curr_symbol", rffi.CCHARP), + ("currency_symbol", rffi.CCHARP), # Local currency symbol. + ("mon_decimal_point", rffi.CCHARP), # Decimal point character. + ("mon_thousands_sep", rffi.CCHARP), # Thousands separator. + ("mon_grouping", rffi.CCHARP), # Like `grouping' element (above). + ("positive_sign", rffi.CCHARP), # Sign for positive values. + ("negative_sign", rffi.CCHARP), # Sign for negative values. + ("int_frac_digits", rffi.UCHAR), # Int'l fractional digits. + + ("frac_digits", rffi.UCHAR), # Local fractional digits. + ## 1 if currency_symbol precedes a positive value, 0 if succeeds. + ("p_cs_precedes", rffi.UCHAR), + ## 1 iff a space separates currency_symbol from a positive value. + ("p_sep_by_space", rffi.UCHAR), + ## 1 if currency_symbol precedes a negative value, 0 if succeeds. + ("n_cs_precedes", rffi.UCHAR), + ## 1 iff a space separates currency_symbol from a negative value. + ("n_sep_by_space", rffi.UCHAR), + + ## Positive and negative sign positions: + ## 0 Parentheses surround the quantity and currency_symbol. + ## 1 The sign string precedes the quantity and currency_symbol. + ## 2 The sign string follows the quantity and currency_symbol. + ## 3 The sign string immediately precedes the currency_symbol. + ## 4 The sign string immediately follows the currency_symbol. + ("p_sign_posn", rffi.UCHAR), + ("n_sign_posn", rffi.UCHAR), + ]) + + +constants = {} +constant_names = ( + 'LC_CTYPE', + 'LC_NUMERIC', + 'LC_TIME', + 'LC_COLLATE', + 'LC_MONETARY', + 'LC_MESSAGES', + 'LC_ALL', + 'LC_PAPER', + 'LC_NAME', + 'LC_ADDRESS', + 'LC_TELEPHONE', + 'LC_MEASUREMENT', + 'LC_IDENTIFICATION', + 'LC_MIN', + 'LC_MAX', + # from limits.h + 'CHAR_MAX', + ) + +for name in constant_names: + setattr(CConfig, name, platform.DefinedConstantInteger(name)) + +langinfo_names = [] +if HAVE_LANGINFO: + # some of these consts have an additional #ifdef directives + # should we support them? + langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' + 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' + 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' + '_DATE_FMT'.split()) + for i in range(1, 8): + langinfo_names.append("DAY_%d" % i) + langinfo_names.append("ABDAY_%d" % i) + for i in range(1, 13): + langinfo_names.append("MON_%d" % i) + langinfo_names.append("ABMON_%d" % i) + +if sys.platform == 'win32': + langinfo_names.extend('LOCALE_USER_DEFAULT LOCALE_SISO639LANGNAME ' + 'LOCALE_SISO3166CTRYNAME LOCALE_IDEFAULTLANGUAGE ' + ''.split()) + + +for name in langinfo_names: + setattr(CConfig, name, platform.DefinedConstantInteger(name)) + +class cConfig(object): + pass + +for k, v in platform.configure(CConfig).items(): + setattr(cConfig, k, v) + +# needed to export the constants inside and outside. see __init__.py +for name in constant_names: + value = getattr(cConfig, name) + if value is not None: + constants[name] = value + +for name in langinfo_names: + value = getattr(cConfig, name) + if value is not None and sys.platform != 'win32': + constants[name] = value + +locals().update(constants) + +HAVE_BIND_TEXTDOMAIN_CODESET = cConfig.HAVE_BIND_TEXTDOMAIN_CODESET + +def external(name, args, result, calling_conv='c'): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv=calling_conv) + +_setlocale = external('setlocale', [rffi.INT, rffi.CCHARP], rffi.CCHARP) + +def setlocale(category, locale): + _setlocale(rffi.cast(rffi.INT, category), locale) Added: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 19:52:44 2010 @@ -0,0 +1,9 @@ + +from pypy.rlib.rlocale import setlocale, LC_CTYPE + +def test_setlocale(): + oldlocale = setlocale(LC_CTYPE, None) + try: + pass + finally: + setlocale(LC_CTYPE, oldlocale) From fijal at codespeak.net Fri Apr 16 21:36:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 21:36:22 +0200 (CEST) Subject: [pypy-svn] r73815 - pypy/trunk/pypy/rlib/test Message-ID: <20100416193622.12BD1282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 21:36:20 2010 New Revision: 73815 Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py Log: Test if setting locale really worked. Skip if there is no polish locale Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rlocale.py (original) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 21:36:20 2010 @@ -1,9 +1,19 @@ -from pypy.rlib.rlocale import setlocale, LC_CTYPE +# -*- coding: utf-8 -*- -def test_setlocale(): - oldlocale = setlocale(LC_CTYPE, None) - try: - pass - finally: - setlocale(LC_CTYPE, oldlocale) +import py +import locale as cpython_locale +from pypy.rlib.rlocale import setlocale, LC_ALL, LocaleError + +class TestLocale(object): + def setup_class(cls): + try: + cls.oldlocale = setlocale(LC_ALL, "pl_PL.utf8") + except LocaleError: + py.test.skip("polish locale unsupported") + + def teardown_class(cls): + setlocale(LC_ALL, cls.oldlocale) + + def test_setlocale(self): + assert u"?".isupper() From fijal at codespeak.net Fri Apr 16 21:36:36 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 21:36:36 +0200 (CEST) Subject: [pypy-svn] r73816 - pypy/trunk/pypy/rlib/test Message-ID: <20100416193636.B8BDE282BAD@codespeak.net> Author: fijal Date: Fri Apr 16 21:36:35 2010 New Revision: 73816 Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py Log: rename function Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rlocale.py (original) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 21:36:35 2010 @@ -15,5 +15,5 @@ def teardown_class(cls): setlocale(LC_ALL, cls.oldlocale) - def test_setlocale(self): + def test_setlocale_worked(self): assert u"?".isupper() From fijal at codespeak.net Fri Apr 16 21:47:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 21:47:33 +0200 (CEST) Subject: [pypy-svn] r73817 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100416194733.AAAA4282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 21:47:32 2010 New Revision: 73817 Modified: pypy/trunk/pypy/rlib/rlocale.py pypy/trunk/pypy/rlib/test/test_rlocale.py Log: Finish setlocale Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 21:47:32 2010 @@ -149,4 +149,10 @@ _setlocale = external('setlocale', [rffi.INT, rffi.CCHARP], rffi.CCHARP) def setlocale(category, locale): - _setlocale(rffi.cast(rffi.INT, category), locale) + if cConfig.LC_MAX is not None: + if not cConfig.LC_MIN <= category <= cConfig.LC_MAX: + raise LocaleError("invalid locale category") + ll_result = _setlocale(rffi.cast(rffi.INT, category), locale) + if not ll_result: + raise LocaleError("unsupported locale setting") + return rffi.charp2str(ll_result) Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rlocale.py (original) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 21:47:32 2010 @@ -17,3 +17,5 @@ def test_setlocale_worked(self): assert u"?".isupper() + raises(LocaleError, setlocale, LC_ALL, "bla bla bla") + raises(LocaleError, setlocale, 1234455, None) From fijal at codespeak.net Fri Apr 16 22:24:48 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 22:24:48 +0200 (CEST) Subject: [pypy-svn] r73818 - pypy/trunk/pypy/rlib Message-ID: <20100416202448.88601282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 22:24:46 2010 New Revision: 73818 Modified: pypy/trunk/pypy/rlib/rlocale.py Log: provide a custom init for exception Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 22:24:46 2010 @@ -9,7 +9,8 @@ from pypy.rpython.tool import rffi_platform as platform class LocaleError(Exception): - pass + def __init__(self, message): + self.message = message HAVE_LANGINFO = sys.platform != 'win32' HAVE_LIBINTL = sys.platform != 'win32' From fijal at codespeak.net Fri Apr 16 22:45:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 22:45:21 +0200 (CEST) Subject: [pypy-svn] r73819 - in pypy/trunk/pypy/rlib: . test Message-ID: <20100416204521.B4D0C282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 22:45:19 2010 New Revision: 73819 Modified: pypy/trunk/pypy/rlib/rlocale.py pypy/trunk/pypy/rlib/test/test_rlocale.py Log: a couple of primitives Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 22:45:19 2010 @@ -157,3 +157,7 @@ if not ll_result: raise LocaleError("unsupported locale setting") return rffi.charp2str(ll_result) + +isalpha = external('isalpha', [rffi.INT], rffi.INT) +isupper = external('isupper', [rffi.INT], rffi.INT) +islower = external('islower', [rffi.INT], rffi.INT) Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rlocale.py (original) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 22:45:19 2010 @@ -3,7 +3,8 @@ import py import locale as cpython_locale -from pypy.rlib.rlocale import setlocale, LC_ALL, LocaleError +from pypy.rlib.rlocale import setlocale, LC_ALL, LocaleError, isupper, \ + islower, isalpha class TestLocale(object): def setup_class(cls): @@ -19,3 +20,9 @@ assert u"?".isupper() raises(LocaleError, setlocale, LC_ALL, "bla bla bla") raises(LocaleError, setlocale, 1234455, None) + + def test_lower_upper(self): + assert isupper(ord("A")) + assert islower(ord("a")) + assert not isalpha(ord(" ")) + From fijal at codespeak.net Fri Apr 16 22:48:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 22:48:30 +0200 (CEST) Subject: [pypy-svn] r73820 - in pypy/trunk/pypy/rlib: . rsre rsre/test test Message-ID: <20100416204830.75654282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 22:48:28 2010 New Revision: 73820 Removed: pypy/trunk/pypy/rlib/rsre/_rsre_platform.py pypy/trunk/pypy/rlib/rsre/test/test_rsre_platform.py Modified: pypy/trunk/pypy/rlib/rlocale.py pypy/trunk/pypy/rlib/rsre/rsre_char.py pypy/trunk/pypy/rlib/test/test_rlocale.py Log: Add a couple more primitives to rlocale. Use it in rsre (keep the mess contained, eh) Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 22:48:28 2010 @@ -161,3 +161,5 @@ isalpha = external('isalpha', [rffi.INT], rffi.INT) isupper = external('isupper', [rffi.INT], rffi.INT) islower = external('islower', [rffi.INT], rffi.INT) +tolower = external('tolower', [rffi.INT], rffi.INT) +isalnum = external('isalnum', [rffi.INT], rffi.INT) Modified: pypy/trunk/pypy/rlib/rsre/rsre_char.py ============================================================================== --- pypy/trunk/pypy/rlib/rsre/rsre_char.py (original) +++ pypy/trunk/pypy/rlib/rsre/rsre_char.py Fri Apr 16 22:48:28 2010 @@ -2,7 +2,7 @@ Character categories and charsets. """ import sys -from pypy.rlib.rsre._rsre_platform import tolower, isalnum +from pypy.rlib.rlocale import tolower, isalnum from pypy.rlib.unroll import unrolling_iterable # Note: the unicode parts of this module require you to call Modified: pypy/trunk/pypy/rlib/test/test_rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rlocale.py (original) +++ pypy/trunk/pypy/rlib/test/test_rlocale.py Fri Apr 16 22:48:28 2010 @@ -4,7 +4,7 @@ import py import locale as cpython_locale from pypy.rlib.rlocale import setlocale, LC_ALL, LocaleError, isupper, \ - islower, isalpha + islower, isalpha, tolower, isalnum class TestLocale(object): def setup_class(cls): @@ -25,4 +25,6 @@ assert isupper(ord("A")) assert islower(ord("a")) assert not isalpha(ord(" ")) + assert isalnum(ord("1")) + assert tolower(ord("A")) == ord("a") From fijal at codespeak.net Fri Apr 16 23:00:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 23:00:17 +0200 (CEST) Subject: [pypy-svn] r73821 - in pypy/trunk/pypy/module/_locale: . test Message-ID: <20100416210017.CE75C282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 23:00:16 2010 New Revision: 73821 Removed: pypy/trunk/pypy/module/_locale/app_locale.py Modified: pypy/trunk/pypy/module/_locale/__init__.py pypy/trunk/pypy/module/_locale/interp_locale.py pypy/trunk/pypy/module/_locale/test/test_locale.py Log: As most of the stuff depending on platform was moved to rlocale, use it. Besides, clean it up a little (and kill the app-level part, it's a bit pointless) Modified: pypy/trunk/pypy/module/_locale/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_locale/__init__.py (original) +++ pypy/trunk/pypy/module/_locale/__init__.py Fri Apr 16 23:00:16 2010 @@ -1,27 +1,29 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module._locale import interp_locale +from pypy.rlib import rlocale import sys class Module(MixedModule): """Support for POSIX locales.""" interpleveldefs = { - 'setlocale': 'interp_locale.setlocale', - 'localeconv': 'interp_locale.localeconv', - 'strcoll': 'interp_locale.strcoll', - 'strxfrm': 'interp_locale.strxfrm', - } + 'setlocale': 'interp_locale.setlocale', + 'localeconv': 'interp_locale.localeconv', + 'strcoll': 'interp_locale.strcoll', + 'strxfrm': 'interp_locale.strxfrm', + 'Error': 'interp_locale.W_Error', + } if sys.platform == 'win32': interpleveldefs.update({ '_getdefaultlocale': 'interp_locale.getdefaultlocale', }) - if interp_locale.HAVE_LANGINFO: + if rlocale.HAVE_LANGINFO: interpleveldefs.update({ 'nl_langinfo': 'interp_locale.nl_langinfo', }) - if interp_locale.HAVE_LIBINTL: + if rlocale.HAVE_LIBINTL: interpleveldefs.update({ 'gettext': 'interp_locale.gettext', 'dgettext': 'interp_locale.dgettext', @@ -29,18 +31,16 @@ 'textdomain': 'interp_locale.textdomain', 'bindtextdomain': 'interp_locale.bindtextdomain', }) - if interp_locale.HAVE_BIND_TEXTDOMAIN_CODESET: + if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET: interpleveldefs.update({ 'bind_textdomain_codeset':'interp_locale.bind_textdomain_codeset', }) appleveldefs = { - 'Error': 'app_locale.Error', - '_fixup_ulcase': 'app_locale._fixup_ulcase', } def buildloaders(cls): - for constant, value in interp_locale.constants.iteritems(): + for constant, value in rlocale.constants.iteritems(): Module.interpleveldefs[constant] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Fri Apr 16 23:00:16 2010 @@ -1,188 +1,72 @@ from pypy.rpython.tool import rffi_platform as platform -from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import rposix from pypy.rlib.rarithmetic import intmask from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib import rlocale +from pypy.module.exceptions.interp_exceptions import _new_exception, W_Exception +from pypy.rpython.lltypesystem import lltype, rffi -import sys - -HAVE_LANGINFO = sys.platform != 'win32' -HAVE_LIBINTL = sys.platform != 'win32' - -class CConfig: - includes = ['locale.h', 'limits.h'] - if HAVE_LANGINFO: - includes += ['langinfo.h'] - if HAVE_LIBINTL: - includes += ['libintl.h'] - if sys.platform == 'win32': - includes += ['windows.h'] - _compilation_info_ = ExternalCompilationInfo( - includes=includes, - ) - HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') - lconv = platform.Struct("struct lconv", [ - # Numeric (non-monetary) information. - ("decimal_point", rffi.CCHARP), # Decimal point character. - ("thousands_sep", rffi.CCHARP), # Thousands separator. - - ## Each element is the number of digits in each group; - ## elements with higher indices are farther left. - ## An element with value CHAR_MAX means that no further grouping is done. - ## An element with value 0 means that the previous element is used - ## for all groups farther left. */ - ("grouping", rffi.CCHARP), - - ## Monetary information. - - ## First three chars are a currency symbol from ISO 4217. - ## Fourth char is the separator. Fifth char is '\0'. - ("int_curr_symbol", rffi.CCHARP), - ("currency_symbol", rffi.CCHARP), # Local currency symbol. - ("mon_decimal_point", rffi.CCHARP), # Decimal point character. - ("mon_thousands_sep", rffi.CCHARP), # Thousands separator. - ("mon_grouping", rffi.CCHARP), # Like `grouping' element (above). - ("positive_sign", rffi.CCHARP), # Sign for positive values. - ("negative_sign", rffi.CCHARP), # Sign for negative values. - ("int_frac_digits", rffi.UCHAR), # Int'l fractional digits. - - ("frac_digits", rffi.UCHAR), # Local fractional digits. - ## 1 if currency_symbol precedes a positive value, 0 if succeeds. - ("p_cs_precedes", rffi.UCHAR), - ## 1 iff a space separates currency_symbol from a positive value. - ("p_sep_by_space", rffi.UCHAR), - ## 1 if currency_symbol precedes a negative value, 0 if succeeds. - ("n_cs_precedes", rffi.UCHAR), - ## 1 iff a space separates currency_symbol from a negative value. - ("n_sep_by_space", rffi.UCHAR), - - ## Positive and negative sign positions: - ## 0 Parentheses surround the quantity and currency_symbol. - ## 1 The sign string precedes the quantity and currency_symbol. - ## 2 The sign string follows the quantity and currency_symbol. - ## 3 The sign string immediately precedes the currency_symbol. - ## 4 The sign string immediately follows the currency_symbol. - ("p_sign_posn", rffi.UCHAR), - ("n_sign_posn", rffi.UCHAR), - ]) - - -constants = {} -constant_names = ( - 'LC_CTYPE', - 'LC_NUMERIC', - 'LC_TIME', - 'LC_COLLATE', - 'LC_MONETARY', - 'LC_MESSAGES', - 'LC_ALL', - 'LC_PAPER', - 'LC_NAME', - 'LC_ADDRESS', - 'LC_TELEPHONE', - 'LC_MEASUREMENT', - 'LC_IDENTIFICATION', - 'LC_MIN', - 'LC_MAX', - # from limits.h - 'CHAR_MAX', - ) - -for name in constant_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -langinfo_names = [] -if HAVE_LANGINFO: - # some of these consts have an additional #ifdef directives - # should we support them? - langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' - 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' - '_DATE_FMT'.split()) - for i in range(1, 8): - langinfo_names.append("DAY_%d" % i) - langinfo_names.append("ABDAY_%d" % i) - for i in range(1, 13): - langinfo_names.append("MON_%d" % i) - langinfo_names.append("ABMON_%d" % i) - -if sys.platform == 'win32': - langinfo_names.extend('LOCALE_USER_DEFAULT LOCALE_SISO639LANGNAME ' - 'LOCALE_SISO3166CTRYNAME LOCALE_IDEFAULTLANGUAGE ' - ''.split()) - - -for name in langinfo_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -class cConfig(object): - pass - -for k, v in platform.configure(CConfig).items(): - setattr(cConfig, k, v) +W_Error = _new_exception('Error', W_Exception, 'locale error') -# needed to export the constants inside and outside. see __init__.py -for name in constant_names: - value = getattr(cConfig, name) - if value is not None: - constants[name] = value - -for name in langinfo_names: - value = getattr(cConfig, name) - if value is not None and sys.platform != 'win32': - constants[name] = value - -locals().update(constants) - -HAVE_BIND_TEXTDOMAIN_CODESET = cConfig.HAVE_BIND_TEXTDOMAIN_CODESET - -def external(name, args, result, calling_conv='c'): - return rffi.llexternal(name, args, result, - compilation_info=CConfig._compilation_info_, - calling_conv=calling_conv) +import sys def make_error(space, msg): - w_module = space.getbuiltinmodule('_locale') - w_exception_class = space.getattr(w_module, space.wrap('Error')) - w_exception = space.call_function(w_exception_class, space.wrap(msg)) - return OperationError(w_exception_class, w_exception) + return OperationError(space.gettypeobject(W_Error.typedef), space.wrap(msg)) -_setlocale = external('setlocale', [rffi.INT, rffi.CCHARP], rffi.CCHARP) +def rewrap_error(space, e): + return OperationError(space.gettypeobject(W_Error.typedef), + space.wrap(e.message)) + +def _fixup_ulcase(space): + stringmod = space.call_function( + space.getattr(space.getbuiltinmodule('__builtin__'), + space.wrap('__import__')), space.wrap('string')) + # create uppercase map string + ul = [] + for c in xrange(256): + if rlocale.isupper(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('uppercase'), space.wrap(''.join(ul))) + + # create lowercase string + ul = [] + for c in xrange(256): + if rlocale.islower(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('lowercase'), space.wrap(''.join(ul))) + + # create letters string + ul = [] + for c in xrange(256): + if rlocale.isalpha(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('letters'), space.wrap(''.join(ul))) def setlocale(space, category, w_locale=None): "(integer,string=None) -> string. Activates/queries locale processing." - if cConfig.LC_MAX is not None: - if not cConfig.LC_MIN <= category <= cConfig.LC_MAX: - raise make_error(space, "invalid locale category") - if space.is_w(w_locale, space.w_None) or w_locale is None: - result = _setlocale(rffi.cast(rffi.INT, category), None) - if not result: - raise make_error(space, "locale query failed") + locale = None else: - locale = rffi.str2charp(space.str_w(w_locale)) + locale = space.str_w(w_locale) + try: + result = rlocale.setlocale(category, locale) + except rlocale.LocaleError, e: + raise rewrap_error(space, e) + + # record changes to LC_CTYPE + if category in (rlocale.LC_CTYPE, rlocale.LC_ALL): + _fixup_ulcase(space) - result = _setlocale(rffi.cast(rffi.INT, category), locale) - if not result: - raise make_error(space, "unsupported locale setting") - - # record changes to LC_CTYPE - if category in (LC_CTYPE, LC_ALL): - w_module = space.getbuiltinmodule('_locale') - w_fun = space.getattr(w_module, space.wrap('_fixup_ulcase')) - space.call_function(w_fun) - - return space.wrap(rffi.charp2str(result)) + return space.wrap(result) setlocale.unwrap_spec = [ObjSpace, int, W_Root] -_lconv = lltype.Ptr(cConfig.lconv) -_localeconv = external('localeconv', [], _lconv) +_lconv = lltype.Ptr(rlocale.cConfig.lconv) +_localeconv = rlocale.external('localeconv', [], _lconv) def _w_copy_grouping(space, text): groups = [ space.wrap(ord(group)) for group in text ] @@ -238,8 +122,8 @@ localeconv.unwrap_spec = [ObjSpace] -_strcoll = external('strcoll', [rffi.CCHARP, rffi.CCHARP], rffi.INT) -_wcscoll = external('wcscoll', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) +_strcoll = rlocale.external('strcoll', [rffi.CCHARP, rffi.CCHARP], rffi.INT) +_wcscoll = rlocale.external('wcscoll', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) def strcoll(space, w_s1, w_s2): "string,string -> int. Compares two strings according to the locale." @@ -264,7 +148,7 @@ strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] -_strxfrm = external('strxfrm', +_strxfrm = rlocale.external('strxfrm', [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T) def strxfrm(space, s): @@ -287,15 +171,15 @@ strxfrm.unwrap_spec = [ObjSpace, str] -if HAVE_LANGINFO: +if rlocale.HAVE_LANGINFO: nl_item = rffi.INT - _nl_langinfo = external('nl_langinfo', [nl_item], rffi.CCHARP) + _nl_langinfo = rlocale.external('nl_langinfo', [nl_item], rffi.CCHARP) def nl_langinfo(space, key): """nl_langinfo(key) -> string Return the value for the locale information associated with key.""" - if key in constants.values(): + if key in rlocale.constants.values(): result = _nl_langinfo(rffi.cast(nl_item, key)) return space.wrap(rffi.charp2str(result)) raise OperationError(space.w_ValueError, @@ -306,8 +190,8 @@ #___________________________________________________________________ # HAVE_LIBINTL dependence -if HAVE_LIBINTL: - _gettext = external('gettext', [rffi.CCHARP], rffi.CCHARP) +if rlocale.HAVE_LIBINTL: + _gettext = rlocale.external('gettext', [rffi.CCHARP], rffi.CCHARP) def gettext(space, msg): """gettext(msg) -> string @@ -316,7 +200,7 @@ gettext.unwrap_spec = [ObjSpace, str] - _dgettext = external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) + _dgettext = rlocale.external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def dgettext(space, w_domain, msg): """dgettext(domain, msg) -> string @@ -332,7 +216,7 @@ dgettext.unwrap_spec = [ObjSpace, W_Root, str] - _dcgettext = external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT], + _dcgettext = rlocale.external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT], rffi.CCHARP) def dcgettext(space, w_domain, msg, category): @@ -353,7 +237,7 @@ dcgettext.unwrap_spec = [ObjSpace, W_Root, str, int] - _textdomain = external('textdomain', [rffi.CCHARP], rffi.CCHARP) + _textdomain = rlocale.external('textdomain', [rffi.CCHARP], rffi.CCHARP) def textdomain(space, w_domain): """textdomain(domain) -> string @@ -370,7 +254,7 @@ textdomain.unwrap_spec = [ObjSpace, W_Root] - _bindtextdomain = external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP], + _bindtextdomain = rlocale.external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def bindtextdomain(space, domain, w_dir): @@ -392,10 +276,10 @@ bindtextdomain.unwrap_spec = [ObjSpace, str, W_Root] - _bind_textdomain_codeset = external('bind_textdomain_codeset', + _bind_textdomain_codeset = rlocale.external('bind_textdomain_codeset', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) - if HAVE_BIND_TEXTDOMAIN_CODESET: + if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET: def bind_textdomain_codeset(space, domain, w_codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" @@ -422,10 +306,10 @@ if sys.platform == 'win32': from pypy.rlib import rwin32 LCID = LCTYPE = rwin32.DWORD - GetACP = external('GetACP', + GetACP = rlocale.external('GetACP', [], rffi.INT, calling_conv='win') - GetLocaleInfo = external('GetLocaleInfoA', + GetLocaleInfo = rlocale.external('GetLocaleInfoA', [LCID, LCTYPE, rwin32.LPSTR, rffi.INT], rffi.INT, calling_conv='win') Modified: pypy/trunk/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/test/test_locale.py (original) +++ pypy/trunk/pypy/module/_locale/test/test_locale.py Fri Apr 16 23:00:16 2010 @@ -7,9 +7,9 @@ def setup_class(cls): cls.space = space = gettestobjspace(usemodules=['_locale']) if sys.platform != 'win32': - cls.w_language_en = cls.space.wrap("en_US") - cls.w_language_utf8 = cls.space.wrap("en_US.UTF-8") - cls.w_language_pl = cls.space.wrap("pl_PL.UTF-8") + cls.w_language_en = cls.space.wrap("C") + cls.w_language_utf8 = cls.space.wrap("en_US.utf8") + cls.w_language_pl = cls.space.wrap("pl_PL.utf8") cls.w_encoding_pl = cls.space.wrap("utf-8") else: cls.w_language_en = cls.space.wrap("English_US") @@ -118,11 +118,11 @@ assert string.lowercase == lcase assert string.uppercase == ucase - if self.language_en != self.language_utf8: - _locale.setlocale(_locale.LC_ALL, self.language_en) + _locale.setlocale(_locale.LC_ALL, self.language_en) - assert string.lowercase != lcase - assert string.uppercase != ucase + # the asserts below are just plain wrong + # assert string.lowercase != lcase + # assert string.uppercase != ucase def test_localeconv(self): import _locale From fijal at codespeak.net Fri Apr 16 23:12:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 23:12:16 +0200 (CEST) Subject: [pypy-svn] r73822 - in pypy/trunk/pypy: module/_locale rlib Message-ID: <20100416211216.B21D5282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 23:12:15 2010 New Revision: 73822 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py pypy/trunk/pypy/rlib/rlocale.py Log: Move nl_langinfo to rlocale Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Fri Apr 16 23:12:15 2010 @@ -172,18 +172,16 @@ strxfrm.unwrap_spec = [ObjSpace, str] if rlocale.HAVE_LANGINFO: - nl_item = rffi.INT - _nl_langinfo = rlocale.external('nl_langinfo', [nl_item], rffi.CCHARP) def nl_langinfo(space, key): """nl_langinfo(key) -> string Return the value for the locale information associated with key.""" - if key in rlocale.constants.values(): - result = _nl_langinfo(rffi.cast(nl_item, key)) - return space.wrap(rffi.charp2str(result)) - raise OperationError(space.w_ValueError, - space.wrap("unsupported langinfo constant")) + try: + return space.wrap(rlocale.nl_langinfo(key)) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("unsupported langinfo constant")) nl_langinfo.unwrap_spec = [ObjSpace, int] Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Fri Apr 16 23:12:15 2010 @@ -163,3 +163,11 @@ islower = external('islower', [rffi.INT], rffi.INT) tolower = external('tolower', [rffi.INT], rffi.INT) isalnum = external('isalnum', [rffi.INT], rffi.INT) + +if HAVE_LANGINFO: + _nl_langinfo = external('nl_langinfo', [rffi.INT], rffi.CCHARP) + + def nl_langinfo(key): + if key in constants.values(): + return rffi.charp2str(_nl_langinfo(rffi.cast(rffi.INT, key))) + raise ValueError From fijal at codespeak.net Fri Apr 16 23:19:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 23:19:18 +0200 (CEST) Subject: [pypy-svn] r73823 - in pypy/trunk/pypy/module/sys: . test Message-ID: <20100416211918.431FD282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 23:19:15 2010 New Revision: 73823 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/module/sys/app.py pypy/trunk/pypy/module/sys/interp_encoding.py pypy/trunk/pypy/module/sys/test/test_sysmodule.py Log: Add getfilesystemencoding using rlocale Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Fri Apr 16 23:19:15 2010 @@ -68,13 +68,14 @@ 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', -} + 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', + } + appleveldefs = { 'excepthook' : 'app.excepthook', '__excepthook__' : 'app.excepthook', 'exit' : 'app.exit', 'exitfunc' : 'app.exitfunc', - 'getfilesystemencoding' : 'app.getfilesystemencoding', 'callstats' : 'app.callstats', 'copyright' : 'app.copyright_str', } Modified: pypy/trunk/pypy/module/sys/app.py ============================================================================== --- pypy/trunk/pypy/module/sys/app.py (original) +++ pypy/trunk/pypy/module/sys/app.py Fri Apr 16 23:19:15 2010 @@ -26,18 +26,6 @@ #import __builtin__ -def getfilesystemencoding(): - """Return the encoding used to convert Unicode filenames in - operating system filenames. - """ - if sys.platform == "win32": - encoding = "mbcs" - elif sys.platform == "darwin": - encoding = "utf-8" - else: - encoding = None - return encoding - def callstats(): """Not implemented.""" return None Modified: pypy/trunk/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/trunk/pypy/module/sys/interp_encoding.py (original) +++ pypy/trunk/pypy/module/sys/interp_encoding.py Fri Apr 16 23:19:15 2010 @@ -1,3 +1,6 @@ +import sys +from pypy.rlib import rlocale + def getdefaultencoding(space): """Return the current default string encoding used by the Unicode implementation.""" @@ -22,3 +25,27 @@ w_encoder = space.getitem(w_functuple, space.wrap(0)) space.sys.w_default_encoder = w_encoder # cache it return w_encoder + +def getfilesystemencoding(space): + """Return the encoding used to convert Unicode filenames in + operating system filenames. + """ + if sys.platform == "win32": + encoding = "mbcs" + elif sys.platform == "darwin": + encoding = "utf-8" + else: + encoding = None + # CPython does this at startup time, I don't thing it matter that much + if rlocale.HAVE_LANGINFO and rlocale.CODESET: + oldlocale = rlocale.setlocale(rlocale.LC_CTYPE, None) + rlocale.setlocale(rlocale.LC_CTYPE, "") + loc_codeset = rlocale.nl_langinfo(rlocale.CODESET) + if loc_codeset: + codecmod = space.getbuiltinmodule('_codecs') + w_res = space.call_function(space.getattr(codecmod, + space.wrap('lookup')), + space.wrap(loc_codeset)) + if space.is_true(w_res): + encoding = loc_codeset + return space.wrap(encoding) Modified: pypy/trunk/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/trunk/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/trunk/pypy/module/sys/test/test_sysmodule.py Fri Apr 16 23:19:15 2010 @@ -3,6 +3,7 @@ from pypy.conftest import option from py.test import raises from pypy.interpreter.gateway import app2interp_temp +import sys def init_globals_via_builtins_hack(space): space.appexec([], """(): @@ -24,6 +25,7 @@ def setup_class(cls): cls.w_appdirect = cls.space.wrap(option.runappdirect) + cls.w_filesystemenc = cls.space.wrap(sys.getfilesystemencoding()) def test_sys_in_modules(self): import sys @@ -108,6 +110,10 @@ assert isinstance(sys.stderr, file) assert isinstance(sys.stdin, file) + def test_getfilesystemencoding(self): + import sys + assert sys.getfilesystemencoding() == self.filesystemenc + class AppTestSysModulePortedFromCPython: def setup_class(cls): From fijal at codespeak.net Fri Apr 16 23:41:37 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 23:41:37 +0200 (CEST) Subject: [pypy-svn] r73824 - in pypy/trunk/pypy: module/sys objspace/std Message-ID: <20100416214137.CACB0282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 23:41:36 2010 New Revision: 73824 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/module/sys/interp_encoding.py pypy/trunk/pypy/objspace/std/objspace.py Log: Store filesystemencoding on space.sys on objspace runtime initialization Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Fri Apr 16 23:41:36 2010 @@ -10,6 +10,7 @@ self.recursionlimit = 100 self.w_default_encoder = None self.defaultencoding = "ascii" + self.filesystemencoding = None interpleveldefs = { '__name__' : '(space.wrap("sys"))', Modified: pypy/trunk/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/trunk/pypy/module/sys/interp_encoding.py (original) +++ pypy/trunk/pypy/module/sys/interp_encoding.py Fri Apr 16 23:41:36 2010 @@ -26,10 +26,7 @@ space.sys.w_default_encoder = w_encoder # cache it return w_encoder -def getfilesystemencoding(space): - """Return the encoding used to convert Unicode filenames in - operating system filenames. - """ +def _getfilesystemencoding(space): if sys.platform == "win32": encoding = "mbcs" elif sys.platform == "darwin": @@ -48,4 +45,10 @@ space.wrap(loc_codeset)) if space.is_true(w_res): encoding = loc_codeset - return space.wrap(encoding) + return encoding + +def getfilesystemencoding(space): + """Return the encoding used to convert Unicode filenames in + operating system filenames. + """ + return space.wrap(space.sys.filesystemencoding) Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Fri Apr 16 23:41:36 2010 @@ -367,6 +367,11 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t + def startup(self): + from pypy.module.sys.interp_encoding import _getfilesystemencoding + ObjSpace.startup(self) + self.sys.filesystemencoding = _getfilesystemencoding(self) + def sliceindices(self, w_slice, w_length): if isinstance(w_slice, W_SliceObject): a, b, c = w_slice.indices3(self, self.int_w(w_length)) From fijal at codespeak.net Fri Apr 16 23:44:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 16 Apr 2010 23:44:44 +0200 (CEST) Subject: [pypy-svn] r73825 - in pypy/trunk/pypy: module/sys objspace/std Message-ID: <20100416214444.7C2F2282B9C@codespeak.net> Author: fijal Date: Fri Apr 16 23:44:43 2010 New Revision: 73825 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/objspace/std/objspace.py Log: Move filesystemencoding init to sysmodule.init Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Fri Apr 16 23:44:43 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError +from pypy.module.sys.interp_encoding import _getfilesystemencoding class Module(MixedModule): """Sys Builtin Module. """ @@ -86,6 +87,9 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) + def init(self, space): + self.filesystemencoding = _getfilesystemencoding(space) + def getmodule(self, name): space = self.space w_modules = self.get('modules') Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Fri Apr 16 23:44:43 2010 @@ -367,11 +367,6 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def startup(self): - from pypy.module.sys.interp_encoding import _getfilesystemencoding - ObjSpace.startup(self) - self.sys.filesystemencoding = _getfilesystemencoding(self) - def sliceindices(self, w_slice, w_length): if isinstance(w_slice, W_SliceObject): a, b, c = w_slice.indices3(self, self.int_w(w_length)) From benjamin at codespeak.net Sat Apr 17 17:53:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 17:53:02 +0200 (CEST) Subject: [pypy-svn] r73826 - pypy/trunk/pypy/interpreter Message-ID: <20100417155302.529BD282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 17:53:00 2010 New Revision: 73826 Modified: pypy/trunk/pypy/interpreter/function.py Log: remove print complaining about duplicate function ids It doesn't seem to help debug anything useful and just clutters output. Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Sat Apr 17 17:53:00 2010 @@ -235,13 +235,9 @@ def _freeze_(self): from pypy.interpreter.gateway import BuiltinCode if isinstance(self.code, BuiltinCode): - identifier = self.code.identifier - if Function._all.get(identifier, self) is not self: - print "builtin code identifier %s used twice: %s and %s" % ( - identifier, self, Function._all[identifier]) # we have been seen by other means so rtyping should not choke # on us - Function._all[identifier] = self + Function._all[self.code.identifier] = self return False def find(identifier): From fijal at codespeak.net Sat Apr 17 19:12:50 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 19:12:50 +0200 (CEST) Subject: [pypy-svn] r73827 - in pypy/trunk/pypy: module/sys objspace/std Message-ID: <20100417171250.1E6B3282B90@codespeak.net> Author: fijal Date: Sat Apr 17 19:12:48 2010 New Revision: 73827 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/objspace/std/objspace.py Log: give up on being nice and move initialization to StdObjSpace.startup. The sys module init is called way too early, during objspace initialization Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Sat Apr 17 19:12:48 2010 @@ -87,9 +87,6 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def init(self, space): - self.filesystemencoding = _getfilesystemencoding(space) - def getmodule(self, name): space = self.space w_modules = self.get('modules') Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Sat Apr 17 19:12:48 2010 @@ -84,6 +84,13 @@ if self.config.objspace.std.withtproxy: transparent.setup(self) + def startup(self): + from pypy.module.sys.interp_encoding import _getfilesystemencoding + + ObjSpace.startup(self) + self.sys.filesystemencoding = _getfilesystemencoding(self) + + def get_builtin_types(self): return self.builtin_types From fijal at codespeak.net Sat Apr 17 19:19:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 19:19:57 +0200 (CEST) Subject: [pypy-svn] r73828 - pypy/trunk/pypy/module/sys Message-ID: <20100417171957.8DFFD282B90@codespeak.net> Author: fijal Date: Sat Apr 17 19:19:56 2010 New Revision: 73828 Modified: pypy/trunk/pypy/module/sys/interp_encoding.py Log: move this to module-level, an attempt to fix translation Modified: pypy/trunk/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/trunk/pypy/module/sys/interp_encoding.py (original) +++ pypy/trunk/pypy/module/sys/interp_encoding.py Sat Apr 17 19:19:56 2010 @@ -26,14 +26,15 @@ space.sys.w_default_encoder = w_encoder # cache it return w_encoder +if sys.platform == "win32": + base_encoding = "mbcs" +elif sys.platform == "darwin": + base_encoding = "utf-8" +else: + base_encoding = None + def _getfilesystemencoding(space): - if sys.platform == "win32": - encoding = "mbcs" - elif sys.platform == "darwin": - encoding = "utf-8" - else: - encoding = None - # CPython does this at startup time, I don't thing it matter that much + encoding = base_encoding if rlocale.HAVE_LANGINFO and rlocale.CODESET: oldlocale = rlocale.setlocale(rlocale.LC_CTYPE, None) rlocale.setlocale(rlocale.LC_CTYPE, "") From fijal at codespeak.net Sat Apr 17 19:28:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 19:28:28 +0200 (CEST) Subject: [pypy-svn] r73829 - pypy/trunk/pypy/module/sys Message-ID: <20100417172828.9B621282B90@codespeak.net> Author: fijal Date: Sat Apr 17 19:28:27 2010 New Revision: 73829 Modified: pypy/trunk/pypy/module/sys/__init__.py Log: remove unnecessary import Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Sat Apr 17 19:28:27 2010 @@ -1,6 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError -from pypy.module.sys.interp_encoding import _getfilesystemencoding class Module(MixedModule): """Sys Builtin Module. """ From fijal at codespeak.net Sat Apr 17 20:13:10 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 20:13:10 +0200 (CEST) Subject: [pypy-svn] r73830 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20100417181310.87E51282B90@codespeak.net> Author: fijal Date: Sat Apr 17 20:13:08 2010 New Revision: 73830 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_gateway.py Log: Add a path_w method that would encode unicode using filesystemencoding and add that to gateway Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Sat Apr 17 20:13:08 2010 @@ -1095,6 +1095,18 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) + def path_w(space, w_obj): + """ Like str_w, but if the object is unicode, encode it using + filesystemencoding + """ + filesystemencoding = space.sys.filesystemencoding + if (filesystemencoding and + space.is_true(space.isinstance(w_obj, space.w_unicode))): + w_obj = space.call_function(space.getattr(w_obj, + space.wrap('encode')), + space.wrap(filesystemencoding)) + return space.str_w(w_obj) + def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sat Apr 17 20:13:08 2010 @@ -137,6 +137,9 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_path(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -238,6 +241,9 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) + def visit_path(self, typ): + self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) + def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -365,6 +371,9 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) + def visit_path(self, typ): + self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) + def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) Modified: pypy/trunk/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_gateway.py (original) +++ pypy/trunk/pypy/interpreter/test/test_gateway.py Sat Apr 17 20:13:08 2010 @@ -1,3 +1,6 @@ + +# -*- coding: utf-8 -*- + from pypy.conftest import gettestobjspace from pypy.interpreter import gateway from pypy.interpreter import argument @@ -451,6 +454,15 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) + def test_interp2app_unwrap_spec_path(self): + space = self.space + def g(space, p): + return p + + app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) + w_app_g = space.wrap(app_g) + w_res = space.call_function(w_app_g, space.wrap(u"?")) + def test_interp2app_classmethod(self): space = self.space w = space.wrap From fijal at codespeak.net Sat Apr 17 20:16:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 20:16:11 +0200 (CEST) Subject: [pypy-svn] r73831 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20100417181611.917E9282B90@codespeak.net> Author: fijal Date: Sat Apr 17 20:16:10 2010 New Revision: 73831 Modified: pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: use 'path' for stat and lstat Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Sat Apr 17 20:16:10 2010 @@ -180,7 +180,7 @@ raise wrap_oserror(space, e, path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, str] +stat.unwrap_spec = [ObjSpace, 'path'] def lstat(space, path): "Like stat(path), but do no follow symbolic links." @@ -190,7 +190,7 @@ raise wrap_oserror(space, e, path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, str] +lstat.unwrap_spec = [ObjSpace, 'path'] class StatState(object): def __init__(self, space): Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Sat Apr 17 20:16:10 2010 @@ -1,3 +1,6 @@ + +# -*- coding: utf-8 -*- + from pypy.objspace.std import StdObjSpace from pypy.tool.udir import udir from pypy.conftest import gettestobjspace @@ -140,6 +143,13 @@ st = self.posix.lstat(".") assert stat.S_ISDIR(st.st_mode) + def test_stat_unicode(self): + # test that passing unicode would not raise UnicodeDecodeError + try: + self.posix.stat(u"?") + except OSError: + pass + def test_stat_exception(self): import sys, errno for fn in [self.posix.stat, self.posix.lstat]: From benjamin at codespeak.net Sat Apr 17 20:23:07 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:23:07 +0200 (CEST) Subject: [pypy-svn] r73832 - pypy/trunk/pypy/objspace/std Message-ID: <20100417182307.D5893282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:23:06 2010 New Revision: 73832 Modified: pypy/trunk/pypy/objspace/std/frame.py Log: must use getdict() to get module dictionary Modified: pypy/trunk/pypy/objspace/std/frame.py ============================================================================== --- pypy/trunk/pypy/objspace/std/frame.py (original) +++ pypy/trunk/pypy/objspace/std/frame.py Sat Apr 17 20:23:06 2010 @@ -75,7 +75,7 @@ if w_value is None: builtins = f.get_builtin() assert isinstance(builtins, Module) - w_builtin_dict = builtins.w_dict + w_builtin_dict = builtins.getdict() assert isinstance(w_builtin_dict, W_DictMultiObject) w_value = w_builtin_dict.get_builtin_indexed(num) if w_value is None: From benjamin at codespeak.net Sat Apr 17 20:26:41 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:26:41 +0200 (CEST) Subject: [pypy-svn] r73833 - in pypy/trunk/pypy: module/sys objspace/std Message-ID: <20100417182641.C78A7282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:26:39 2010 New Revision: 73833 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/objspace/std/objspace.py Log: move file system encoding initialization back to sys's initialization Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Sat Apr 17 20:26:39 2010 @@ -1,5 +1,6 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError +from pypy.module.sys.interp_encoding import _getfilesystemencoding class Module(MixedModule): """Sys Builtin Module. """ @@ -86,6 +87,9 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) + def init(self, space): + self.filesystemencoding = _getfilesystemencoding(space) + def getmodule(self, name): space = self.space w_modules = self.get('modules') Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Sat Apr 17 20:26:39 2010 @@ -84,13 +84,6 @@ if self.config.objspace.std.withtproxy: transparent.setup(self) - def startup(self): - from pypy.module.sys.interp_encoding import _getfilesystemencoding - - ObjSpace.startup(self) - self.sys.filesystemencoding = _getfilesystemencoding(self) - - def get_builtin_types(self): return self.builtin_types From benjamin at codespeak.net Sat Apr 17 20:36:58 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:36:58 +0200 (CEST) Subject: [pypy-svn] r73834 - pypy/trunk/pypy/interpreter Message-ID: <20100417183658.42264282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:36:56 2010 New Revision: 73834 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py Log: only check for duplicate ids during translation Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Sat Apr 17 20:36:56 2010 @@ -237,6 +237,8 @@ if isinstance(self.code, BuiltinCode): # we have been seen by other means so rtyping should not choke # on us + assert self.code.identitifer not in Function._all, ("duplicate " + "function ids") Function._all[self.code.identifier] = self return False Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sat Apr 17 20:36:56 2010 @@ -801,8 +801,6 @@ defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) - if not space.config.translating: # for tests and py.py - fn._freeze_() if gateway.as_classmethod: fn = ClassMethod(space.wrap(fn)) return fn From benjamin at codespeak.net Sat Apr 17 20:48:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:48:02 +0200 (CEST) Subject: [pypy-svn] r73835 - pypy/trunk/pypy/objspace/std Message-ID: <20100417184802.7870A282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:48:00 2010 New Revision: 73835 Modified: pypy/trunk/pypy/objspace/std/frame.py Log: cleanup imports in frame.py Modified: pypy/trunk/pypy/objspace/std/frame.py ============================================================================== --- pypy/trunk/pypy/objspace/std/frame.py (original) +++ pypy/trunk/pypy/objspace/std/frame.py Sat Apr 17 20:48:00 2010 @@ -6,14 +6,16 @@ from pypy.interpreter import pyframe, pyopcode, function from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module.__builtin__ import OPTIMIZED_BUILTINS, Module +from pypy.objspace.std import intobject, smallintobject from pypy.objspace.std.multimethod import FailedToImplement +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.listobject import W_ListObject class BaseFrame(pyframe.PyFrame): """These opcodes are always overridden.""" def LIST_APPEND(f, oparg, next_instr): - from pypy.objspace.std.listobject import W_ListObject w = f.popvalue() v = f.popvalue() if type(v) is W_ListObject: @@ -23,13 +25,12 @@ def small_int_BINARY_ADD(f, oparg, next_instr): - from pypy.objspace.std.smallintobject import (W_SmallIntObject, - add__SmallInt_SmallInt) w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_SmallIntObject and type(w_2) is W_SmallIntObject: + if (type(w_1) is smallintobject.W_SmallIntObject and + type(w_2) is smallintobject.W_SmallIntObject): try: - w_result = add__SmallInt_SmallInt(f.space, w_1, w_2) + w_result = smallintobject.add__SmallInt_SmallInt(f.space, w_1, w_2) except FailedToImplement: w_result = f.space.add(w_1, w_2) else: @@ -38,12 +39,12 @@ def int_BINARY_ADD(f, oparg, next_instr): - from pypy.objspace.std.intobject import W_IntObject, add__Int_Int w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_IntObject and type(w_2) is W_IntObject: + if (type(w_1) is intobject.W_IntObject and + type(w_2) is intobject.W_IntObject): try: - w_result = add__Int_Int(f.space, w_1, w_2) + w_result = intobject.add__Int_Int(f.space, w_1, w_2) except FailedToImplement: w_result = f.space.add(w_1, w_2) else: @@ -52,11 +53,9 @@ def list_BINARY_SUBSCR(f, oparg, next_instr): - from pypy.objspace.std.intobject import W_IntObject - from pypy.objspace.std.listobject import W_ListObject w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_ListObject and type(w_2) is W_IntObject: + if type(w_1) is W_ListObject and type(w_2) is intobject.W_IntObject: try: w_result = w_1.wrappeditems[w_2.intval] except IndexError: @@ -67,7 +66,6 @@ f.pushvalue(w_result) def CALL_LIKELY_BUILTIN(f, oparg, next_instr): - from pypy.objspace.std.dictmultiobject import W_DictMultiObject w_globals = f.w_globals num = oparg >> 8 assert isinstance(w_globals, W_DictMultiObject) @@ -113,12 +111,12 @@ unrolling_compare_ops = unrolling_iterable(enumerate(compare_table)) def fast_COMPARE_OP(f, testnum, next_instr): - from pypy.objspace.std.intobject import W_IntObject w_2 = f.popvalue() w_1 = f.popvalue() w_result = None - if (type(w_2) is W_IntObject and type(w_1) is W_IntObject - and testnum < len(compare_table)): + if (type(w_2) is intobject.W_IntObject and + type(w_1) is intobject.W_IntObject and + testnum < len(compare_table)): for i, attr in unrolling_compare_ops: if i == testnum: op = getattr(operator, attr) From benjamin at codespeak.net Sat Apr 17 20:50:14 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:50:14 +0200 (CEST) Subject: [pypy-svn] r73836 - pypy/trunk/pypy/interpreter Message-ID: <20100417185014.695BE282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:50:13 2010 New Revision: 73836 Modified: pypy/trunk/pypy/interpreter/function.py Log: factor out common code Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Sat Apr 17 20:50:13 2010 @@ -237,9 +237,9 @@ if isinstance(self.code, BuiltinCode): # we have been seen by other means so rtyping should not choke # on us - assert self.code.identitifer not in Function._all, ("duplicate " - "function ids") - Function._all[self.code.identifier] = self + identifier = self.code.identifier + assert identifier not in Function._all, ("duplicate function ids") + Function._all[identifier] = self return False def find(identifier): From benjamin at codespeak.net Sat Apr 17 20:55:42 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:55:42 +0200 (CEST) Subject: [pypy-svn] r73837 - pypy/trunk/pypy/interpreter Message-ID: <20100417185542.892A9282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:55:40 2010 New Revision: 73837 Modified: pypy/trunk/pypy/interpreter/function.py Log: handle the same code being used twice Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Sat Apr 17 20:55:40 2010 @@ -238,7 +238,8 @@ # we have been seen by other means so rtyping should not choke # on us identifier = self.code.identifier - assert identifier not in Function._all, ("duplicate function ids") + assert Function._all.get(identifier, self) is self, ("duplicate " + "function ids") Function._all[identifier] = self return False From benjamin at codespeak.net Sat Apr 17 20:58:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 20:58:02 +0200 (CEST) Subject: [pypy-svn] r73838 - pypy/trunk/pypy/module/sys Message-ID: <20100417185802.EED06282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 20:58:01 2010 New Revision: 73838 Modified: pypy/trunk/pypy/module/sys/__init__.py Log: use startup() over init() Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Sat Apr 17 20:58:01 2010 @@ -87,7 +87,7 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def init(self, space): + def startup(self, space): self.filesystemencoding = _getfilesystemencoding(space) def getmodule(self, name): From benjamin at codespeak.net Sat Apr 17 21:10:47 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 21:10:47 +0200 (CEST) Subject: [pypy-svn] r73839 - pypy/trunk/pypy/interpreter/test Message-ID: <20100417191047.77FB9282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 21:10:29 2010 New Revision: 73839 Modified: pypy/trunk/pypy/interpreter/test/test_gateway.py Log: fix for when file system encoding is horrible (like ascii) Modified: pypy/trunk/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_gateway.py (original) +++ pypy/trunk/pypy/interpreter/test/test_gateway.py Sat Apr 17 21:10:29 2010 @@ -454,13 +454,14 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) - def test_interp2app_unwrap_spec_path(self): + def test_interp2app_unwrap_spec_path(self, monkeypatch): space = self.space def g(space, p): return p app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) w_app_g = space.wrap(app_g) + monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") w_res = space.call_function(w_app_g, space.wrap(u"?")) def test_interp2app_classmethod(self): From benjamin at codespeak.net Sat Apr 17 21:13:34 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 21:13:34 +0200 (CEST) Subject: [pypy-svn] r73840 - pypy/trunk/pypy/interpreter Message-ID: <20100417191334.73CF0282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 21:13:25 2010 New Revision: 73840 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py Log: simplify with call_method Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Sat Apr 17 21:13:25 2010 @@ -1102,9 +1102,8 @@ filesystemencoding = space.sys.filesystemencoding if (filesystemencoding and space.is_true(space.isinstance(w_obj, space.w_unicode))): - w_obj = space.call_function(space.getattr(w_obj, - space.wrap('encode')), - space.wrap(filesystemencoding)) + w_obj = space.call_method(w_obj, "encode", + space.wrap(filesystemencoding)) return space.str_w(w_obj) def bool_w(self, w_obj): From benjamin at codespeak.net Sat Apr 17 21:27:49 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 17 Apr 2010 21:27:49 +0200 (CEST) Subject: [pypy-svn] r73841 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20100417192749.0C818282B90@codespeak.net> Author: benjamin Date: Sat Apr 17 21:27:47 2010 New Revision: 73841 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/test/test_objspace.py Log: introduce space.isinstance_w shortuct Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Sat Apr 17 21:27:47 2010 @@ -877,6 +877,9 @@ w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + def isinstance_w(self, w_obj, w_type): + return self.is_true(self.isinstance(w_obj, w_type)) + # The code below only works # for the simple case (new-style instance). # These methods are patched with the full logic by the __builtin__ Modified: pypy/trunk/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_objspace.py (original) +++ pypy/trunk/pypy/interpreter/test/test_objspace.py Sat Apr 17 21:27:47 2010 @@ -12,7 +12,18 @@ INT32_MAX = 2147483648 -class TestObjSpace: +class TestObjSpace: + + def test_isinstance(self): + space = self.space + w_i = space.wrap(4) + w_result = space.isinstance(w_i, space.w_int) + assert space.is_true(w_result) + assert space.isinstance_w(w_i, space.w_int) + w_result = space.isinstance(w_i, space.w_str) + assert not space.is_true(w_result) + assert not space.isinstance_w(w_i, space.w_str) + def test_newlist(self): w = self.space.wrap l = range(10) From fijal at codespeak.net Sat Apr 17 22:28:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 22:28:44 +0200 (CEST) Subject: [pypy-svn] r73842 - in pypy/branch/decouple-host-opcodes/pypy: interpreter objspace/flow objspace/flow/test objspace/std Message-ID: <20100417202844.60B6C282B90@codespeak.net> Author: fijal Date: Sat Apr 17 22:28:41 2010 New Revision: 73842 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py Log: A major overhaul - kill PyPyFrame and HostFrame and instead have a difference in FlowObjSpace, while keeping PyFrame a PyPyFrame Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Sat Apr 17 22:28:41 2010 @@ -258,12 +258,9 @@ self.actionflag.register_action(self.frame_trace_action) from pypy.interpreter.pycode import cpython_magic, default_magic - from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame self.our_magic = default_magic self.host_magic = cpython_magic # can be overridden to a subclass - self.FrameClass = PyPyFrame - self.HostFrameClass = HostPyFrame if self.config.objspace.logbytecodes: self.bytecodecounts = [0] * 256 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyframe.py Sat Apr 17 22:28:41 2010 @@ -136,8 +136,7 @@ # the following 'assert' is an annotation hint: it hides from # the annotator all methods that are defined in PyFrame but # overridden in the {,Host}FrameClass subclasses of PyFrame. - assert (isinstance(self, self.space.FrameClass) - or isinstance(self, self.space.HostFrameClass)) + assert isinstance(self, self.space.FrameClass) executioncontext = self.space.getexecutioncontext() executioncontext.enter(self) try: @@ -631,13 +630,6 @@ return space.wrap(self.builtin is not space.builtin) return space.w_False - -class PyPyFrame(PyFrame): - pass - -class HostPyFrame(PyFrame): - pass - # ____________________________________________________________ def get_block_class(opname): Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/pyopcode.py Sat Apr 17 22:28:41 2010 @@ -65,6 +65,12 @@ # for logbytecode: last_opcode = -1 + bytecode_spec = bytecode_spec + opcode_method_names = bytecode_spec.method_names + opcodedesc = bytecode_spec.opcodedesc + opdescmap = bytecode_spec.opdescmap + HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT + ### opcode dispatch ### def dispatch(self, pycode, next_instr, ec): @@ -1016,17 +1022,6 @@ (ofs, ord(c), name) ) STOP_CODE = MISSING_OPCODE - - -class __extend__(pyframe.PyPyFrame): - """ - Execution of PyPy opcodes (mostly derived from Python 2.5.2). - """ - bytecode_spec = bytecode_spec - opcode_method_names = bytecode_spec.method_names - opcodedesc = bytecode_spec.opcodedesc - opdescmap = bytecode_spec.opdescmap - HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT def BUILD_MAP(self, itemcount, next_instr): if itemcount != 0: @@ -1037,94 +1032,6 @@ def STORE_MAP(self, zero, next_instr): raise BytecodeCorruption -# Need to cast 2.7's "named struct" to a plain struct -host_version_info = tuple(sys.version_info) - -class __extend__(pyframe.HostPyFrame): - """ - Execution of host (CPython) opcodes. - """ - bytecode_spec = host_bytecode_spec - opcode_method_names = host_bytecode_spec.method_names - opcodedesc = host_bytecode_spec.opcodedesc - opdescmap = host_bytecode_spec.opdescmap - HAVE_ARGUMENT = host_bytecode_spec.HAVE_ARGUMENT - - def BUILD_MAP(self, itemcount, next_instr): - if host_version_info >= (2, 6): - # We could pre-allocate a dict here - # but for the moment this code is not translated. - pass - else: - if itemcount != 0: - raise BytecodeCorruption - w_dict = self.space.newdict() - self.pushvalue(w_dict) - - def STORE_MAP(self, zero, next_instr): - if host_version_info >= (2, 6): - w_key = self.popvalue() - w_value = self.popvalue() - w_dict = self.peekvalue() - self.space.setitem(w_dict, w_key, w_value) - else: - raise BytecodeCorruption - - def POP_JUMP_IF_FALSE(self, jumpto, next_instr): - w_cond = self.popvalue() - if not self.space.is_true(w_cond): - next_instr = jumpto - return next_instr - - def POP_JUMP_IF_TRUE(self, jumpto, next_instr): - w_cond = self.popvalue() - if self.space.is_true(w_cond): - return jumpto - return next_instr - - def JUMP_IF_FALSE_OR_POP(self, jumpto, next_instr): - w_cond = self.peekvalue() - if not self.space.is_true(w_cond): - return jumpto - self.popvalue() - return next_instr - - def JUMP_IF_TRUE_OR_POP(self, jumpto, next_instr): - w_cond = self.peekvalue() - if self.space.is_true(w_cond): - return jumpto - self.popvalue() - return next_instr - - def LIST_APPEND(self, oparg, next_instr): - w = self.popvalue() - if host_version_info < (2, 7): - v = self.popvalue() - else: - v = self.peekvalue(oparg - 1) - self.space.call_method(v, 'append', w) - - # XXX Unimplemented 2.7 opcodes ---------------- - - # Set literals, set comprehensions - - def BUILD_SET(self, oparg, next_instr): - raise NotImplementedError("BUILD_SET") - - def SET_ADD(self, oparg, next_instr): - raise NotImplementedError("SET_ADD") - - # Dict comprehensions - - def MAP_ADD(self, oparg, next_instr): - raise NotImplementedError("MAP_ADD") - - # `with` statement - - def SETUP_WITH(self, oparg, next_instr): - raise NotImplementedError("SETUP_WITH") - - ### ____________________________________________________________ ### class ExitFrame(Exception): Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/flowcontext.py Sat Apr 17 22:28:41 2010 @@ -1,12 +1,13 @@ import collections from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError -from pypy.interpreter import pyframe, pycode +from pypy.interpreter import pyframe from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState from pypy.rlib import jit - +from pypy.tool.stdlib_opcode import host_bytecode_spec +import sys class OperationThatShouldNotBePropagatedError(OperationError): pass @@ -219,12 +220,8 @@ # create an empty frame suitable for the code object # while ignoring any operation like the creation of the locals dict self.recorder = [] - if self.code.magic == pycode.cpython_magic: - frame = FlowSpaceHostPyFrame(self.space, self.code, - self.w_globals, self.closure) - else: - frame = FlowSpacePyPyFrame(self.space, self.code, - self.w_globals, self.closure) + frame = FlowSpaceFrame(self.space, self.code, + self.w_globals, self.closure) frame.last_instr = 0 return frame @@ -402,7 +399,91 @@ if w_v.value is oldvalue: stack_items_w[i] = w_new -class FlowSpaceFrameBase(object): +class FlowSpaceFrame(pyframe.PyFrame): + """ + Execution of host (CPython) opcodes. + """ + bytecode_spec = host_bytecode_spec + opcode_method_names = host_bytecode_spec.method_names + opcodedesc = host_bytecode_spec.opcodedesc + opdescmap = host_bytecode_spec.opdescmap + HAVE_ARGUMENT = host_bytecode_spec.HAVE_ARGUMENT + + def BUILD_MAP(self, itemcount, next_instr): + if sys.version_info >= (2, 6): + # We could pre-allocate a dict here + # but for the moment this code is not translated. + pass + else: + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + + def STORE_MAP(self, zero, next_instr): + if sys.version_info >= (2, 6): + w_key = self.popvalue() + w_value = self.popvalue() + w_dict = self.peekvalue() + self.space.setitem(w_dict, w_key, w_value) + else: + raise BytecodeCorruption + + def POP_JUMP_IF_FALSE(self, jumpto, next_instr): + w_cond = self.popvalue() + if not self.space.is_true(w_cond): + next_instr = jumpto + return next_instr + + def POP_JUMP_IF_TRUE(self, jumpto, next_instr): + w_cond = self.popvalue() + if self.space.is_true(w_cond): + return jumpto + return next_instr + + def JUMP_IF_FALSE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if not self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def JUMP_IF_TRUE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def LIST_APPEND(self, oparg, next_instr): + w = self.popvalue() + if sys.version_info < (2, 7): + v = self.popvalue() + else: + v = self.peekvalue(oparg - 1) + self.space.call_method(v, 'append', w) + + # XXX Unimplemented 2.7 opcodes ---------------- + + # Set literals, set comprehensions + + def BUILD_SET(self, oparg, next_instr): + raise NotImplementedError("BUILD_SET") + + def SET_ADD(self, oparg, next_instr): + raise NotImplementedError("SET_ADD") + + # Dict comprehensions + + def MAP_ADD(self, oparg, next_instr): + raise NotImplementedError("MAP_ADD") + + # `with` statement + + def SETUP_WITH(self, oparg, next_instr): + raise NotImplementedError("SETUP_WITH") + + def make_arguments(self, nargs): return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) def argument_factory(self, *args): @@ -414,10 +495,3 @@ raise operr return pyframe.PyFrame.handle_operation_error(self, ec, operr, *args, **kwds) - -class FlowSpacePyPyFrame(FlowSpaceFrameBase, pyframe.PyPyFrame): - pass - -class FlowSpaceHostPyFrame(FlowSpaceFrameBase, pyframe.HostPyFrame): - pass - Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/objspace.py Sat Apr 17 22:28:41 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode, cpython_code_signature from pypy.interpreter.module import Module from pypy.interpreter.error import OperationError +from pypy.interpreter import pyframe from pypy.objspace.flow.model import * from pypy.objspace.flow import flowcontext from pypy.objspace.flow.operation import FunctionByName @@ -24,7 +25,6 @@ else: type_with_bad_introspection = type(complex.real.__get__) - # ______________________________________________________________________ class FlowObjSpace(ObjSpace): """NOT_RPYTHON. @@ -35,6 +35,7 @@ full_exceptions = False do_imports_immediately = True + FrameClass = flowcontext.FlowSpaceFrame def initialize(self): import __builtin__ @@ -85,14 +86,6 @@ return Constant({}) return self.do_operation('newdict') - def createframe(self, code, w_globals, closure=None): - magic = code.magic - if magic == self.host_magic: - return self.HostFrameClass(self, code, w_globals, closure) - elif magic == self.our_magic: - return self.FrameClass(self, code, w_globals, closure) - raise ValueError("bad magic %s" % magic) - def newtuple(self, args_w): try: content = [self.unwrap(w_arg) for w_arg in args_w] Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/flow/test/test_objspace.py Sat Apr 17 22:28:41 2010 @@ -826,34 +826,24 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - from pypy.objspace.flow import flowcontext - old_class = flowcontext.FlowSpaceHostPyFrame - try: - # HACK: we will replace a host-generated code object with a - # pypy-generated code object, but it will carry the host's - # magic number (since it's generated with the host's code.new). - flowcontext.FlowSpaceHostPyFrame = flowcontext.FlowSpacePyPyFrame - class X: - def m(self): - return 3 - - def f(): - x = X() - return x.m() - - # this code is generated by pypy-c when compiling above f - pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' - new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) - f2 = new.function(new_c, locals(), 'f') - import dis - dis.dis(f2) - - graph = self.codetest(f2) - all_ops = self.all_operations(graph) - assert all_ops['simple_call'] == 2 - assert all_ops['getattr'] == 1 - finally: - flowcontext.FlowSpaceHostPyFrame = old_class + py.test.skip("XXX this needs to be fixed, maybe") + class X: + def m(self): + return 3 + + def f(): + x = X() + return x.m() + + # this code is generated by pypy-c when compiling above f + pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' + new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) + f2 = new.function(new_c, locals(), 'f') + + graph = self.codetest(f2) + all_ops = self.all_operations(graph) + assert all_ops['simple_call'] == 2 + assert all_ops['getattr'] == 1 def test_generator(self): def f(): Modified: pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/objspace/std/objspace.py Sat Apr 17 22:28:41 2010 @@ -47,9 +47,8 @@ # setup all the object types and implementations self.model = model.StdTypeModel(self.config) - from pypy.interpreter.pyframe import PyPyFrame, HostPyFrame - self.FrameClass = frame.build_frame(self, PyPyFrame) - self.HostFrameClass = frame.build_frame(self, HostPyFrame) + from pypy.interpreter.pyframe import PyFrame + self.FrameClass = frame.build_frame(self, PyFrame) if self.config.objspace.std.withrope: self.StringObjectCls = W_RopeObject From fijal at codespeak.net Sat Apr 17 23:06:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:06:20 +0200 (CEST) Subject: [pypy-svn] r73843 - pypy/branch/decouple-host-opcodes/py/impl/code Message-ID: <20100417210620.3B7A7282BDE@codespeak.net> Author: fijal Date: Sat Apr 17 23:06:18 2010 New Revision: 73843 Modified: pypy/branch/decouple-host-opcodes/py/impl/code/_assertionold.py Log: both places don't need import sys Modified: pypy/branch/decouple-host-opcodes/py/impl/code/_assertionold.py ============================================================================== --- pypy/branch/decouple-host-opcodes/py/impl/code/_assertionold.py (original) +++ pypy/branch/decouple-host-opcodes/py/impl/code/_assertionold.py Sat Apr 17 23:06:18 2010 @@ -447,7 +447,6 @@ def check(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) expr = parse(s, 'eval') @@ -518,7 +517,6 @@ def run(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) module = Interpretable(parse(s, 'exec').node) From fijal at codespeak.net Sat Apr 17 23:23:06 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:23:06 +0200 (CEST) Subject: [pypy-svn] r73844 - in pypy/trunk/pypy/module/pypyjit: . test Message-ID: <20100417212306.8B1AF282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:23:04 2010 New Revision: 73844 Modified: pypy/trunk/pypy/module/pypyjit/policy.py pypy/trunk/pypy/module/pypyjit/test/test_policy.py Log: Don't look into rlocale Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Sat Apr 17 23:23:04 2010 @@ -24,8 +24,7 @@ # gc_id operation if func.__name__ == 'id__ANY': return False - if mod == 'pypy.rlib.rbigint': - #if func.__name__ == '_bigint_true_divide': + if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff return False Modified: pypy/trunk/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_policy.py Sat Apr 17 23:23:04 2010 @@ -10,6 +10,10 @@ from pypy.rlib.rbigint import rbigint assert not pypypolicy.look_inside_function(rbigint.lt.im_func) +def test_rlocale(): + from pypy.rlib.rlocale import setlocale + assert not pypypolicy.look_inside_function(setlocale) + def test_geninterp(): d = {'_geninterp_': True} exec """def f(): From fijal at codespeak.net Sat Apr 17 23:34:56 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:34:56 +0200 (CEST) Subject: [pypy-svn] r73845 - pypy/branch/decouple-host-opcodes/pypy/interpreter Message-ID: <20100417213456.3F1A3282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:34:54 2010 New Revision: 73845 Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Log: Give up. I can't convince pytestsupport to not use these APIs that way. Modified: pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/decouple-host-opcodes/pypy/interpreter/baseobjspace.py Sat Apr 17 23:34:54 2010 @@ -934,7 +934,8 @@ expression = compiler.compile(expression, '?', 'eval', 0, hidden_applevel=hidden_applevel) if isinstance(expression, types.CodeType): - raise Exception("space.eval(cpycode) Should not be used") + # XXX only used by appsupport + expression = PyCode._from_code(self, expression) if not isinstance(expression, PyCode): raise TypeError, 'space.eval(): expected a string, code or PyCode object' return expression.exec_code(self, w_globals, w_locals) @@ -951,7 +952,8 @@ statement = compiler.compile(statement, filename, 'exec', 0, hidden_applevel=hidden_applevel) if isinstance(statement, types.CodeType): - raise Exception("space.eval(cpycode) Should not be used") + # XXX only used by appsupport + statement = PyCode._from_code(self, statement) if not isinstance(statement, PyCode): raise TypeError, 'space.exec_(): expected a string, code or PyCode object' w_key = self.wrap('__builtins__') From fijal at codespeak.net Sat Apr 17 23:35:56 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:35:56 +0200 (CEST) Subject: [pypy-svn] r73846 - pypy/trunk/pypy/config Message-ID: <20100417213556.714C0282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:35:54 2010 New Revision: 73846 Modified: pypy/trunk/pypy/config/translationoption.py Log: Kill the comment. I don't know how to forbid that yet, but well. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Sat Apr 17 23:35:54 2010 @@ -100,7 +100,7 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - suggests=[("translation.gc", "hybrid"), # or "boehm" + suggests=[("translation.gc", "hybrid"), ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), ChoiceOption("jit_backend", "choose the backend for the JIT", From fijal at codespeak.net Sat Apr 17 23:45:44 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:45:44 +0200 (CEST) Subject: [pypy-svn] r73847 - in pypy/trunk: . dotviewer lib-python py/impl/code pypy pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/test pypy/interpreter/pyparser pypy/interpreter/test pypy/jit/backend/cli/test pypy/jit/backend/llvm/test pypy/jit/backend/x86/test pypy/jit/metainterp pypy/module/__builtin__ pypy/module/_codecs pypy/module/exceptions pypy/objspace/flow pypy/objspace/flow/test pypy/objspace/std pypy/objspace/std/test pypy/rlib/test pypy/tool pypy/tool/pytest pypy/translator pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 Message-ID: <20100417214544.4F414282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:45:41 2010 New Revision: 73847 Added: pypy/trunk/pypy/translator/goal/test2/test_targetpypy.py - copied unchanged from r73846, pypy/branch/decouple-host-opcodes/pypy/translator/goal/test2/test_targetpypy.py Modified: pypy/trunk/ (props changed) pypy/trunk/dotviewer/ (props changed) pypy/trunk/lib-python/ (props changed) pypy/trunk/py/impl/code/_assertionold.py pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/interpreter/astcompiler/assemble.py pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/interpreter/pycompiler.py pypy/trunk/pypy/interpreter/pyframe.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/interpreter/pyparser/pyparse.py pypy/trunk/pypy/interpreter/test/test_appinterp.py pypy/trunk/pypy/interpreter/test/test_compiler.py pypy/trunk/pypy/jit/backend/cli/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/metainterp/logger.py (props changed) pypy/trunk/pypy/module/__builtin__/app_inspect.py pypy/trunk/pypy/module/_codecs/app_codecs.py pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/flow/flowcontext.py pypy/trunk/pypy/objspace/flow/objspace.py pypy/trunk/pypy/objspace/flow/specialcase.py pypy/trunk/pypy/objspace/flow/test/test_objspace.py pypy/trunk/pypy/objspace/std/frame.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/test/test_obj.py pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/rlib/test/test_rcoroutine.py (props changed) pypy/trunk/pypy/tool/pytest/appsupport.py pypy/trunk/pypy/tool/sourcetools.py pypy/trunk/pypy/tool/stdlib_opcode.py pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) pypy/trunk/pypy/translator/geninterplevel.py pypy/trunk/pypy/translator/goal/nanos.py pypy/trunk/pypy/translator/goal/targetpypystandalone.py Log: (antoine, fijal) Merge decouple-host-opcodes. This branch separates cpython bytecode from pypy bytecode as much as possible. We no longer support translating frozen CPython bytecode. Should make pypy work on CPython 2.7. Modified: pypy/trunk/py/impl/code/_assertionold.py ============================================================================== --- pypy/trunk/py/impl/code/_assertionold.py (original) +++ pypy/trunk/py/impl/code/_assertionold.py Sat Apr 17 23:45:41 2010 @@ -447,7 +447,6 @@ def check(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) expr = parse(s, 'eval') @@ -518,7 +517,6 @@ def run(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) module = Interpretable(parse(s, 'exec').node) Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/assemble.py Sat Apr 17 23:45:41 2010 @@ -400,7 +400,8 @@ self.first_lineno, lnotab, free_names, - cell_names) + cell_names, + self.compile_info.hidden_applevel) def _list_from_dict(d, offset=0): Modified: pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Sat Apr 17 23:45:41 2010 @@ -22,11 +22,15 @@ """ def run(self, source): + import sys source = str(py.code.Source(source)) space = self.space code = compile_with_astcompiler(source, 'exec', space) - print - code.dump() + # 2.7 bytecode is too different, the standard `dis` module crashes + # when trying to display pypy (2.5-like) bytecode. + if sys.version_info < (2, 7): + print + code.dump() w_dict = space.newdict() code.exec_code(space, w_dict, w_dict) return w_dict @@ -39,7 +43,12 @@ pyco_expr = PyCode._from_code(space, co_expr) w_res = pyco_expr.exec_code(space, w_dict, w_dict) res = space.str_w(space.repr(w_res)) - assert res == repr(expected) + if not isinstance(expected, float): + assert res == repr(expected) + else: + # Float representation can vary a bit between interpreter + # versions, compare the numbers instead. + assert eval(res) == expected def simple_test(self, source, evalexpr, expected): w_g = self.run(source) Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Sat Apr 17 23:45:41 2010 @@ -13,7 +13,7 @@ from pypy.rlib.timer import DummyTimer, Timer from pypy.rlib.rarithmetic import r_uint from pypy.rlib import jit -import os, sys +import os, sys, py __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -257,8 +257,10 @@ self.actionflag.register_action(self.user_del_action) self.actionflag.register_action(self.frame_trace_action) - from pypy.interpreter.pyframe import PyFrame - self.FrameClass = PyFrame # can be overridden to a subclass + from pypy.interpreter.pycode import cpython_magic, default_magic + self.our_magic = default_magic + self.host_magic = cpython_magic + # can be overridden to a subclass if self.config.objspace.logbytecodes: self.bytecodecounts = [0] * 256 @@ -931,23 +933,30 @@ import types from pypy.interpreter.pycode import PyCode if isinstance(expression, str): - expression = compile(expression, '?', 'eval') + compiler = self.createcompiler() + expression = compiler.compile(expression, '?', 'eval', 0, + hidden_applevel=hidden_applevel) if isinstance(expression, types.CodeType): - expression = PyCode._from_code(self, expression, - hidden_applevel=hidden_applevel) + # XXX only used by appsupport + expression = PyCode._from_code(self, expression) if not isinstance(expression, PyCode): raise TypeError, 'space.eval(): expected a string, code or PyCode object' return expression.exec_code(self, w_globals, w_locals) - def exec_(self, statement, w_globals, w_locals, hidden_applevel=False): + def exec_(self, statement, w_globals, w_locals, hidden_applevel=False, + filename=None): "NOT_RPYTHON: For internal debugging." import types + if filename is None: + filename = '?' from pypy.interpreter.pycode import PyCode if isinstance(statement, str): - statement = compile(statement, '?', 'exec') + compiler = self.createcompiler() + statement = compiler.compile(statement, filename, 'exec', 0, + hidden_applevel=hidden_applevel) if isinstance(statement, types.CodeType): - statement = PyCode._from_code(self, statement, - hidden_applevel=hidden_applevel) + # XXX only used by appsupport + statement = PyCode._from_code(self, statement) if not isinstance(statement, PyCode): raise TypeError, 'space.exec_(): expected a string, code or PyCode object' w_key = self.wrap('__builtins__') @@ -1176,7 +1185,7 @@ assert source.startswith('('), "incorrect header in:\n%s" % (source,) source = py.code.Source("def anonymous%s\n" % source) w_glob = space.newdict() - space.exec_(source.compile(), w_glob, w_glob) + space.exec_(str(source), w_glob, w_glob) return space.getitem(w_glob, space.wrap('anonymous')) class DummyLock(object): Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sat Apr 17 23:45:41 2010 @@ -819,6 +819,9 @@ # and now for something completely different ... # +class MyStr(str): + pass + class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -828,12 +831,14 @@ hidden_applevel = True - def __init__(self, source, filename = None, modname = '__builtin__'): + def __init__(self, source, filename=None, modname='__builtin__'): + # HAAACK (but a good one) + if filename is None: + f = sys._getframe(1) + filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) + filename.__source__ = py.code.Source(source) self.filename = filename - if self.filename is None: - self.code = py.code.Source(source).compile() - else: - self.code = NiceCompile(self.filename)(source) + self.source = str(py.code.Source(source).deindent()) self.modname = modname # look at the first three lines for a NOT_RPYTHON tag first = "\n".join(source.split("\n", 3)[:3]) @@ -917,8 +922,9 @@ from pypy.interpreter.pycode import PyCode w_glob = space.newdict(module=True) space.setitem(w_glob, space.wrap('__name__'), space.wrap(self.modname)) - space.exec_(self.code, w_glob, w_glob, - hidden_applevel=self.hidden_applevel) + space.exec_(self.source, w_glob, w_glob, + hidden_applevel=self.hidden_applevel, + filename=self.filename) return w_glob # __________ geninterplevel version __________ @@ -938,7 +944,7 @@ from pypy.translator.geninterplevel import translate_as_module import marshal scramble = md5(cls.seed) - scramble.update(marshal.dumps(self.code)) + scramble.update(marshal.dumps(self.source)) key = scramble.hexdigest() initfunc = cls.known_code.get(key) if not initfunc: @@ -959,7 +965,7 @@ if not initfunc: # build it and put it into a file initfunc, newsrc = translate_as_module( - self.code, self.filename, self.modname) + self.source, self.filename, self.modname) fname = cls.cache_path.join(name+".py").strpath f = file(get_tmp_file_name(fname), "w") print >> f, """\ Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Sat Apr 17 23:45:41 2010 @@ -117,6 +117,10 @@ self._compute_flatcall() + def _freeze_(self): + if self.magic == cpython_magic: + raise Exception("CPython host codes should not be rendered") + return False def _init_flags(self): co_code = self.co_code Modified: pypy/trunk/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycompiler.py (original) +++ pypy/trunk/pypy/interpreter/pycompiler.py Sat Apr 17 23:45:41 2010 @@ -151,8 +151,9 @@ e.wrap_info(space)) return mod - def compile(self, source, filename, mode, flags): + def compile(self, source, filename, mode, flags, hidden_applevel=False): from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags) + info = CompileInfo(filename, mode, flags, + hidden_applevel=hidden_applevel) mod = self._compile_to_ast(source, info) return self._compile_ast(mod, info) Modified: pypy/trunk/pypy/interpreter/pyframe.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyframe.py (original) +++ pypy/trunk/pypy/interpreter/pyframe.py Sat Apr 17 23:45:41 2010 @@ -135,7 +135,7 @@ from pypy.rlib import rstack # the following 'assert' is an annotation hint: it hides from # the annotator all methods that are defined in PyFrame but - # overridden in the FrameClass subclass of PyFrame. + # overridden in the {,Host}FrameClass subclasses of PyFrame. assert isinstance(self, self.space.FrameClass) executioncontext = self.space.getexecutioncontext() executioncontext.enter(self) @@ -238,6 +238,8 @@ self.pushvalue(w_value) def peekvalue(self, index_from_top=0): + # NOTE: top of the stack is peekvalue(0). + # Contrast this with CPython where it's PEEK(-1). index_from_top = hint(index_from_top, promote=True) index = self.valuestackdepth + ~index_from_top assert index >= 0, "peek past the bottom of the stack" Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sat Apr 17 23:45:41 2010 @@ -14,9 +14,8 @@ from pypy.rlib import jit, rstackovf from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.tool.stdlib_opcode import (opcodedesc, HAVE_ARGUMENT, - unrolling_opcode_descs, - opcode_method_names) +from pypy.tool.stdlib_opcode import ( + bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) def unaryoperation(operationname): """NOT_RPYTHON""" @@ -66,6 +65,12 @@ # for logbytecode: last_opcode = -1 + bytecode_spec = bytecode_spec + opcode_method_names = bytecode_spec.method_names + opcodedesc = bytecode_spec.opcodedesc + opdescmap = bytecode_spec.opdescmap + HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT + ### opcode dispatch ### def dispatch(self, pycode, next_instr, ec): @@ -180,7 +185,7 @@ probs[opcode] = probs.get(opcode, 0) + 1 self.last_opcode = opcode - if opcode >= HAVE_ARGUMENT: + if opcode >= self.HAVE_ARGUMENT: lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 @@ -188,16 +193,16 @@ else: oparg = 0 - while opcode == opcodedesc.EXTENDED_ARG.index: + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) - if opcode < HAVE_ARGUMENT: + if opcode < self.HAVE_ARGUMENT: raise BytecodeCorruption lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 oparg = (oparg << 16) | (hi << 8) | lo - if opcode == opcodedesc.RETURN_VALUE.index: + if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() block = self.unrollstack(SReturnValue.kind) if block is None: @@ -208,11 +213,11 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == opcodedesc.YIELD_VALUE.index: + if opcode == self.opcodedesc.YIELD_VALUE.index: #self.last_instr = intmask(next_instr - 1) XXX clean up! raise Yield - if opcode == opcodedesc.END_FINALLY.index: + if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): # go on unrolling the stack @@ -225,24 +230,28 @@ next_instr = block.handle(self, unroller) return next_instr - if opcode == opcodedesc.JUMP_ABSOLUTE.index: + if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, next_instr, ec) if we_are_translated(): from pypy.rlib import rstack # for resume points - for opdesc in unrolling_opcode_descs: + for opdesc in unrolling_all_opcode_descs: # static checks to skip this whole case if necessary + if opdesc.bytecode_spec is not self.bytecode_spec: + continue if not opdesc.is_enabled(space): continue - if not hasattr(pyframe.PyFrame, opdesc.methodname): - continue # e.g. for JUMP_ABSOLUTE, implemented above + if opdesc.methodname in ( + 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'END_FINALLY', 'JUMP_ABSOLUTE'): + continue # opcodes implemented above if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) res = meth(oparg, next_instr) - if opdesc.index == opcodedesc.CALL_FUNCTION.index: + if opdesc.index == self.opcodedesc.CALL_FUNCTION.index: rstack.resume_point("dispatch_call", self, co_code, next_instr, ec) # !! warning, for the annotator the next line is not # comparing an int and None - you can't do that. @@ -254,8 +263,23 @@ self.MISSING_OPCODE(oparg, next_instr) else: # when we are not translated, a list lookup is much faster - methodname = opcode_method_names[opcode] - res = getattr(self, methodname)(oparg, next_instr) + methodname = self.opcode_method_names[opcode] + try: + meth = getattr(self, methodname) + except AttributeError: + raise BytecodeCorruption("unimplemented opcode, ofs=%d, code=%d, name=%s" % + (self.last_instr, opcode, methodname)) + try: + res = meth(oparg, next_instr) + except Exception: + if 0: + import dis, sys + print "*** %s at offset %d (%s)" % (sys.exc_info()[0], self.last_instr, methodname) + try: + dis.dis(co_code) + except: + pass + raise if res is not None: next_instr = res @@ -690,26 +714,6 @@ w_list = self.space.newlist(items) self.pushvalue(w_list) - def BUILD_MAP(self, itemcount, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - # We could pre-allocate a dict here - # but for the moment this code is not translated. - pass - else: - if itemcount != 0: - raise BytecodeCorruption - w_dict = self.space.newdict() - self.pushvalue(w_dict) - - def STORE_MAP(self, zero, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - w_key = self.popvalue() - w_value = self.popvalue() - w_dict = self.peekvalue() - self.space.setitem(w_dict, w_key, w_value) - else: - raise BytecodeCorruption - def LOAD_ATTR(self, nameindex, next_instr): "obj.attributename" w_attributename = self.getname_w(nameindex) @@ -1011,14 +1015,22 @@ self.pushvalue(w_result) def MISSING_OPCODE(self, oparg, next_instr): - ofs = next_instr - 1 + ofs = self.last_instr c = self.pycode.co_code[ofs] name = self.pycode.co_name raise BytecodeCorruption("unknown opcode, ofs=%d, code=%d, name=%s" % (ofs, ord(c), name) ) STOP_CODE = MISSING_OPCODE + + def BUILD_MAP(self, itemcount, next_instr): + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + def STORE_MAP(self, zero, next_instr): + raise BytecodeCorruption ### ____________________________________________________________ ### Modified: pypy/trunk/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/pyparse.py Sat Apr 17 23:45:41 2010 @@ -65,14 +65,18 @@ * encoding: The source encoding. * last_future_import: The line number and offset of the last __future__ import. + * hidden_applevel: Will this code unit and sub units be hidden at the + applevel? """ - def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0)): + def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0), + hidden_applevel=False): self.filename = filename self.mode = mode self.encoding = None self.flags = flags self.last_future_import = future_pos + self.hidden_applevel = hidden_applevel _targets = { Modified: pypy/trunk/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/trunk/pypy/interpreter/test/test_appinterp.py Sat Apr 17 23:45:41 2010 @@ -1,6 +1,7 @@ import py from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp, applevelinterp_temp +from pypy.interpreter.error import OperationError def test_execwith_novars(space): val = space.appexec([], """ @@ -18,11 +19,11 @@ assert space.eq_w(val, space.wrap(42)) def test_execwith_compile_error(space): - excinfo = py.test.raises(SyntaxError, space.appexec, [], """ + excinfo = py.test.raises(OperationError, space.appexec, [], """ (): y y """) - assert str(excinfo.value).find('y y') != -1 + assert str(excinfo.value.errorstr(space)).find('y y') != -1 def test_simple_applevel(space): app = appdef("""app(x,y): Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Sat Apr 17 23:45:41 2010 @@ -54,6 +54,14 @@ space.raises_w(space.w_SyntaxError, self.compiler.compile_command, ')', '?', mode, 0) + def test_hidden_applevel(self): + code = self.compiler.compile("def f(x): pass", "", "exec", 0, + True) + assert code.hidden_applevel + for w_const in code.co_consts_w: + if isinstance(w_const, PyCode): + assert code.hidden_applevel + def test_indentation_error(self): space = self.space space.raises_w(space.w_SyntaxError, self.compiler.compile_command, Modified: pypy/trunk/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/trunk/pypy/module/__builtin__/app_inspect.py Sat Apr 17 23:45:41 2010 @@ -64,9 +64,7 @@ if isinstance(obj, types.ModuleType): try: - result = obj.__dict__.keys() - if not isinstance(result, list): - raise TypeError("expected __dict__.keys() to be a list") + result = list(obj.__dict__.keys()) result.sort() return result except AttributeError: Modified: pypy/trunk/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/app_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/app_codecs.py Sat Apr 17 23:45:41 2010 @@ -1,7 +1,3 @@ -# NOT_RPYTHON -# Note: -# This *is* now explicitly RPython. -# Please make sure not to break this. """ @@ -356,6 +352,7 @@ bitsleft = 0 charsleft = 0 surrogate = 0 + startinpos = 0 p = [] errorHandler = None exc = None Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Sat Apr 17 23:45:41 2010 @@ -6,7 +6,8 @@ from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState from pypy.rlib import jit - +from pypy.tool.stdlib_opcode import host_bytecode_spec +import sys class OperationThatShouldNotBePropagatedError(OperationError): pass @@ -399,6 +400,90 @@ stack_items_w[i] = w_new class FlowSpaceFrame(pyframe.PyFrame): + """ + Execution of host (CPython) opcodes. + """ + bytecode_spec = host_bytecode_spec + opcode_method_names = host_bytecode_spec.method_names + opcodedesc = host_bytecode_spec.opcodedesc + opdescmap = host_bytecode_spec.opdescmap + HAVE_ARGUMENT = host_bytecode_spec.HAVE_ARGUMENT + + def BUILD_MAP(self, itemcount, next_instr): + if sys.version_info >= (2, 6): + # We could pre-allocate a dict here + # but for the moment this code is not translated. + pass + else: + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + + def STORE_MAP(self, zero, next_instr): + if sys.version_info >= (2, 6): + w_key = self.popvalue() + w_value = self.popvalue() + w_dict = self.peekvalue() + self.space.setitem(w_dict, w_key, w_value) + else: + raise BytecodeCorruption + + def POP_JUMP_IF_FALSE(self, jumpto, next_instr): + w_cond = self.popvalue() + if not self.space.is_true(w_cond): + next_instr = jumpto + return next_instr + + def POP_JUMP_IF_TRUE(self, jumpto, next_instr): + w_cond = self.popvalue() + if self.space.is_true(w_cond): + return jumpto + return next_instr + + def JUMP_IF_FALSE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if not self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def JUMP_IF_TRUE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def LIST_APPEND(self, oparg, next_instr): + w = self.popvalue() + if sys.version_info < (2, 7): + v = self.popvalue() + else: + v = self.peekvalue(oparg - 1) + self.space.call_method(v, 'append', w) + + # XXX Unimplemented 2.7 opcodes ---------------- + + # Set literals, set comprehensions + + def BUILD_SET(self, oparg, next_instr): + raise NotImplementedError("BUILD_SET") + + def SET_ADD(self, oparg, next_instr): + raise NotImplementedError("SET_ADD") + + # Dict comprehensions + + def MAP_ADD(self, oparg, next_instr): + raise NotImplementedError("MAP_ADD") + + # `with` statement + + def SETUP_WITH(self, oparg, next_instr): + raise NotImplementedError("SETUP_WITH") + + def make_arguments(self, nargs): return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) def argument_factory(self, *args): Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Sat Apr 17 23:45:41 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode, cpython_code_signature from pypy.interpreter.module import Module from pypy.interpreter.error import OperationError +from pypy.interpreter import pyframe from pypy.objspace.flow.model import * from pypy.objspace.flow import flowcontext from pypy.objspace.flow.operation import FunctionByName @@ -24,7 +25,6 @@ else: type_with_bad_introspection = type(complex.real.__get__) - # ______________________________________________________________________ class FlowObjSpace(ObjSpace): """NOT_RPYTHON. @@ -35,6 +35,7 @@ full_exceptions = False do_imports_immediately = True + FrameClass = flowcontext.FlowSpaceFrame def initialize(self): import __builtin__ Modified: pypy/trunk/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/specialcase.py (original) +++ pypy/trunk/pypy/objspace/flow/specialcase.py Sat Apr 17 23:45:41 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import ApplevelClass from pypy.interpreter.error import OperationError from pypy.tool.cache import Cache +import py def sc_import(space, fn, args): args_w, kwds_w = args.unpack() @@ -73,7 +74,7 @@ if app.filename is not None: dic['__file__'] = app.filename dic['__name__'] = app.modname - exec app.code in dic + exec py.code.Source(app.source).compile() in dic return dic _build = staticmethod(_build) Modified: pypy/trunk/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/test/test_objspace.py Sat Apr 17 23:45:41 2010 @@ -837,6 +837,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ + py.test.skip("XXX this needs to be fixed, maybe") class X: def m(self): return 3 Modified: pypy/trunk/pypy/objspace/std/frame.py ============================================================================== --- pypy/trunk/pypy/objspace/std/frame.py (original) +++ pypy/trunk/pypy/objspace/std/frame.py Sat Apr 17 23:45:41 2010 @@ -3,7 +3,8 @@ import operator from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter import pyframe, pyopcode, function +from pypy.interpreter import pyopcode, function +from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module.__builtin__ import OPTIMIZED_BUILTINS, Module from pypy.objspace.std import intobject, smallintobject @@ -12,7 +13,7 @@ from pypy.objspace.std.listobject import W_ListObject -class BaseFrame(pyframe.PyFrame): +class BaseFrame(PyFrame): """These opcodes are always overridden.""" def LIST_APPEND(f, oparg, next_instr): Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Sat Apr 17 23:45:41 2010 @@ -133,7 +133,7 @@ if not we_are_translated() and isinstance(code, CPythonFakeCode): return CPythonFakeFrame(self, code, w_globals) else: - return self.FrameClass(self, code, w_globals, closure) + return ObjSpace.createframe(self, code, w_globals, closure) def gettypefor(self, cls): return self.gettypeobject(cls.typedef) Modified: pypy/trunk/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_obj.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_obj.py Sat Apr 17 23:45:41 2010 @@ -7,11 +7,14 @@ cpython_behavior = (not option.runappdirect or not hasattr(sys, 'pypy_translation_info')) - cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_version = cls.space.wrap(tuple(sys.version_info)) def test_hash_builtin(self): if not self.cpython_behavior: skip("on pypy-c id == hash is not guaranteed") + if self.cpython_version >= (2, 7): + skip("on CPython >= 2.7, id != hash") import sys o = object() assert (hash(o) & sys.maxint) == (id(o) & sys.maxint) @@ -33,7 +36,7 @@ class X(object): pass x = X() - if self.cpython_behavior: + if self.cpython_behavior and self.cpython_version < (2, 7): assert (hash(x) & sys.maxint) == (id(x) & sys.maxint) assert hash(x) == object.__hash__(x) Modified: pypy/trunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/appsupport.py Sat Apr 17 23:45:41 2010 @@ -87,7 +87,7 @@ if debug_excs: self._excinfo = debug_excs[0] - def exconly(self, tryshort=True): + def exconly(self, tryshort=True): return '(application-level) ' + self.operr.errorstr(self.space) def errisinstance(self, exc): @@ -218,7 +218,8 @@ for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) try: - space.exec_(source.compile(), frame.w_globals, w_locals) + space.exec_(str(source), frame.w_globals, w_locals, + filename=__file__) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) Modified: pypy/trunk/pypy/tool/sourcetools.py ============================================================================== --- pypy/trunk/pypy/tool/sourcetools.py (original) +++ pypy/trunk/pypy/tool/sourcetools.py Sat Apr 17 23:45:41 2010 @@ -17,7 +17,7 @@ indentation. The shorter triple quotes are choosen automatically. The result is returned as a 1-tuple.""" - if type(func) is not str: + if not isinstance(func, str): doc = func.__doc__ else: doc = func Modified: pypy/trunk/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/trunk/pypy/tool/stdlib_opcode.py (original) +++ pypy/trunk/pypy/tool/stdlib_opcode.py Sat Apr 17 23:45:41 2010 @@ -1,44 +1,25 @@ +""" +Opcodes PyPy compiles Python source to. +Also gives access to opcodes of the host Python PyPy was bootstrapped with +(module attributes with the `host_` prefix). +""" + # load opcode.py as pythonopcode from our own lib __all__ = ['opmap', 'opname', 'HAVE_ARGUMENT', 'hasconst', 'hasname', 'hasjrel', 'hasjabs', 'haslocal', 'hascompare', 'hasfree', 'cmp_op'] -def load_opcode(): - import py - opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') - d = {} - execfile(str(opcode_path), d) - return d - -opcode_dict = load_opcode() -del load_opcode - -# copy some stuff from opcode.py directly into our globals -for name in __all__: - if name in opcode_dict: - globals()[name] = opcode_dict[name] -globals().update(opmap) -SLICE = opmap["SLICE+0"] -STORE_SLICE = opmap["STORE_SLICE+0"] -DELETE_SLICE = opmap["DELETE_SLICE+0"] - -opcode_method_names = ['MISSING_OPCODE'] * 256 -for name, index in opmap.items(): - opcode_method_names[index] = name.replace('+', '_') - # ____________________________________________________________ # RPython-friendly helpers and structures -from pypy.rlib.unroll import unrolling_iterable - - -class OpcodeDesc(object): - def __init__(self, name, index): +class _BaseOpcodeDesc(object): + def __init__(self, bytecode_spec, name, index, methodname): + self.bytecode_spec = bytecode_spec self.name = name - self.methodname = opcode_method_names[index] + self.methodname = methodname self.index = index - self.hasarg = index >= HAVE_ARGUMENT + self.hasarg = index >= self.HAVE_ARGUMENT def _freeze_(self): return True @@ -64,24 +45,87 @@ return (cmp(self.__class__, other.__class__) or cmp(self.sortkey(), other.sortkey())) -opdescmap = {} + def __str__(self): + return "" % (self.index, self.name, id(self)) + + __repr__ = __str__ + +class _baseopcodedesc: + """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs.""" + pass + + +class BytecodeSpec(object): + """A bunch of mappings describing a bytecode instruction set.""" + + def __init__(self, name, opmap, HAVE_ARGUMENT): + """NOT_RPYTHON.""" + class OpcodeDesc(_BaseOpcodeDesc): + HAVE_ARGUMENT = HAVE_ARGUMENT + class opcodedesc(_baseopcodedesc): + """A namespace mapping OPCODE_NAME to OpcodeDescs.""" + + self.name = name + self.OpcodeDesc = OpcodeDesc + self.opcodedesc = opcodedesc + self.HAVE_ARGUMENT = HAVE_ARGUMENT + # opname -> opcode + self.opmap = opmap + # opcode -> method name + self.method_names = tbl = ['MISSING_OPCODE'] * 256 + # opcode -> opdesc + self.opdescmap = {} + for name, index in opmap.items(): + tbl[index] = methodname = name.replace('+', '_') + desc = OpcodeDesc(self, name, index, methodname) + setattr(self.opcodedesc, name, desc) + self.opdescmap[index] = desc + # fill the ordered opdesc list + self.ordered_opdescs = lst = self.opdescmap.values() + lst.sort() + + def to_globals(self): + """NOT_RPYTHON. Add individual opcodes to the module constants.""" + g = globals() + g.update(self.opmap) + g['SLICE'] = self.opmap["SLICE+0"] + g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] + g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + + def __str__(self): + return "<%s bytecode>" % (self.name,) + + __repr__ = __str__ + + +# Initialization + +from pypy.rlib.unroll import unrolling_iterable + +from opcode import ( + opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) -class opcodedesc: - """A namespace mapping OPCODE_NAME to OpcodeDescs.""" +def load_pypy_opcode(): + import py + opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') + d = {} + execfile(str(opcode_path), d) + for name in __all__: + if name in d: + globals()[name] = d[name] + return d -for name, index in opmap.items(): - desc = OpcodeDesc(name, index) - setattr(opcodedesc, name, desc) - opdescmap[index] = desc - -lst = opdescmap.values() -lst.sort() -unrolling_opcode_descs = unrolling_iterable(lst) - -# Allow non-translated code to interpret the new 2.6 bytecodes -import sys -if sys.version_info >= (2, 6): - import opcode - opcode_method_names[opcode.opmap['STORE_MAP']] = 'STORE_MAP' +load_pypy_opcode() +del load_pypy_opcode -del name, index, desc, lst +bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT) +host_bytecode_spec = BytecodeSpec('host', host_opmap, host_HAVE_ARGUMENT) +bytecode_spec.to_globals() + +opcode_method_names = bytecode_spec.method_names +opcodedesc = bytecode_spec.opcodedesc + +unrolling_all_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs + host_bytecode_spec.ordered_opdescs) +unrolling_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs) Modified: pypy/trunk/pypy/translator/geninterplevel.py ============================================================================== --- pypy/trunk/pypy/translator/geninterplevel.py (original) +++ pypy/trunk/pypy/translator/geninterplevel.py Sat Apr 17 23:45:41 2010 @@ -71,7 +71,7 @@ log = py.log.Producer("geninterp") py.log.setconsumer("geninterp", ansi_log) -GI_VERSION = '1.2.7' # bump this for substantial changes +GI_VERSION = '1.2.8' # bump this for substantial changes # ____________________________________________________________ try: @@ -161,11 +161,11 @@ Constant(OperationError).key: late_OperationError, Constant(Arguments).key: late_Arguments, } - u = UniqueList - self.initcode = u() # list of lines for the module's initxxx() - self.latercode = u() # list of generators generating extra lines - # for later in initxxx() -- for recursive - # objects + self.initcode = UniqueList() # list of lines for the module's initxxx() + self.latercode = UniqueList() + # list of generators generating extra lines + # for later in initxxx() -- for recursive + # objects self.namespace = NameManager() self.namespace.make_reserved_names('__doc__ __args__ space goto') self.globaldecl = [] @@ -1475,10 +1475,7 @@ """ # create something like a module if type(sourcetext) is str: - if filename is None: - code = py.code.Source(sourcetext).compile() - else: - code = NiceCompile(filename)(sourcetext) + code = py.code.Source(sourcetext).compile() else: # assume we got an already compiled source code = sourcetext Modified: pypy/trunk/pypy/translator/goal/nanos.py ============================================================================== --- pypy/trunk/pypy/translator/goal/nanos.py (original) +++ pypy/trunk/pypy/translator/goal/nanos.py Sat Apr 17 23:45:41 2010 @@ -26,7 +26,7 @@ """ from pypy.interpreter.gateway import applevel, ObjSpace, W_Root, interp2app -import os +import os, py app_os_path = applevel(r''' from os.path import dirname, join, abspath, isfile, islink @@ -59,6 +59,6 @@ path_module_for_testing = type(os)("os.path") os_module_for_testing = type(os)("os") os_module_for_testing.path = path_module_for_testing -eval(app_os_path.code, path_module_for_testing.__dict__) -eval(app_os.code, os_module_for_testing.__dict__) +eval(py.code.Source(app_os_path.source).compile(), path_module_for_testing.__dict__) +eval(py.code.Source(app_os.source).compile(), os_module_for_testing.__dict__) Modified: pypy/trunk/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/trunk/pypy/translator/goal/targetpypystandalone.py Sat Apr 17 23:45:41 2010 @@ -233,15 +233,17 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict() - space.exec_(open(filename).read(), w_dict, w_dict) + app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + app.hidden_applevel = False + app.can_use_geninterp = False + w_dict = app.getwdict(space) entry_point = create_entry_point(space, w_dict) return entry_point, None, PyPyAnnotatorPolicy(single_space = space) def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', - 'jitpolicy', + 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) From fijal at codespeak.net Sat Apr 17 23:49:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:49:22 +0200 (CEST) Subject: [pypy-svn] r73848 - pypy/branch/decouple-host-opcodes Message-ID: <20100417214922.685B3282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:49:20 2010 New Revision: 73848 Removed: pypy/branch/decouple-host-opcodes/ Log: Kill merged branch From fijal at codespeak.net Sat Apr 17 23:50:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 17 Apr 2010 23:50:13 +0200 (CEST) Subject: [pypy-svn] r73849 - pypy/trunk/pypy/objspace/flow Message-ID: <20100417215013.BD810282B90@codespeak.net> Author: fijal Date: Sat Apr 17 23:50:11 2010 New Revision: 73849 Modified: pypy/trunk/pypy/objspace/flow/objspace.py Log: I don't think XXX XXX Ha Ha is a very useful comment. The comment below is ok though Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Sat Apr 17 23:50:11 2010 @@ -446,7 +446,6 @@ w_exc_value) def w_KeyboardInterrupt(self): - # XXX XXX Ha Ha # the reason to do this is: if you interrupt the flowing of a function # with the bytecode interpreter will raise an applevel # KeyboardInterrupt and you will get an AttributeError: space does not From benjamin at codespeak.net Sun Apr 18 00:02:07 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 00:02:07 +0200 (CEST) Subject: [pypy-svn] r73850 - in pypy/trunk: dotviewer lib-python pypy pypy/jit/backend/cli/test pypy/jit/backend/llvm/test pypy/jit/backend/x86/test pypy/jit/metainterp pypy/module/exceptions pypy/objspace/std/test pypy/rlib/test pypy/translator/c/test Message-ID: <20100417220207.0D29B282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 00:02:06 2010 New Revision: 73850 Modified: pypy/trunk/dotviewer/ (props changed) pypy/trunk/lib-python/ (props changed) pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/jit/backend/cli/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/metainterp/logger.py (props changed) pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/rlib/test/test_rcoroutine.py (props changed) pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) Log: remove svn:mergeinfo on non-root nodes From fijal at codespeak.net Sun Apr 18 00:10:11 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 18 Apr 2010 00:10:11 +0200 (CEST) Subject: [pypy-svn] r73851 - pypy/trunk/pypy/objspace/flow/test Message-ID: <20100417221011.36D4E282B90@codespeak.net> Author: fijal Date: Sun Apr 18 00:10:09 2010 New Revision: 73851 Modified: pypy/trunk/pypy/objspace/flow/test/test_objspace.py Log: A bit more delicate patching so host will recognize LOOKUP and CALL method bytecodes Modified: pypy/trunk/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/test/test_objspace.py Sun Apr 18 00:10:09 2010 @@ -5,8 +5,10 @@ from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph from pypy.objspace.flow.objspace import FlowObjSpace -from pypy.objspace.flow import objspace +from pypy.objspace.flow import objspace, flowcontext from pypy import conftest +from pypy.tool.stdlib_opcode import bytecode_spec +from pypy.interpreter.pyframe import PyFrame import os import operator @@ -837,24 +839,34 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - py.test.skip("XXX this needs to be fixed, maybe") - class X: - def m(self): - return 3 - - def f(): - x = X() - return x.m() - - # this code is generated by pypy-c when compiling above f - pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' - new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) - f2 = new.function(new_c, locals(), 'f') - - graph = self.codetest(f2) - all_ops = self.all_operations(graph) - assert all_ops['simple_call'] == 2 - assert all_ops['getattr'] == 1 + flow_meth_names = flowcontext.FlowSpaceFrame.opcode_method_names + pyframe_meth_names = PyFrame.opcode_method_names + for name in ['CALL_METHOD', 'LOOKUP_METHOD']: + num = bytecode_spec.opmap[name] + locals()['old_' + name] = flow_meth_names[num] + flow_meth_names[num] = pyframe_meth_names[num] + try: + class X: + def m(self): + return 3 + + def f(): + x = X() + return x.m() + + # this code is generated by pypy-c when compiling above f + pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' + new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) + f2 = new.function(new_c, locals(), 'f') + + graph = self.codetest(f2) + all_ops = self.all_operations(graph) + assert all_ops['simple_call'] == 2 + assert all_ops['getattr'] == 1 + finally: + for name in ['CALL_METHOD', 'LOOKUP_METHOD']: + num = bytecode_spec.opmap[name] + flow_meth_names[num] = locals()['old_' + name] def test_generator(self): def f(): From benjamin at codespeak.net Sun Apr 18 00:15:56 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 00:15:56 +0200 (CEST) Subject: [pypy-svn] r73852 - pypy/trunk/pypy/interpreter Message-ID: <20100417221556.10D82282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 00:15:54 2010 New Revision: 73852 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: wrap import Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun Apr 18 00:15:54 2010 @@ -14,8 +14,9 @@ from pypy.rlib import jit, rstackovf from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.tool.stdlib_opcode import ( - bytecode_spec, host_bytecode_spec, unrolling_all_opcode_descs, opmap, host_opmap) +from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, + unrolling_all_opcode_descs, opmap, + host_opmap) def unaryoperation(operationname): """NOT_RPYTHON""" From benjamin at codespeak.net Sun Apr 18 00:22:30 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 00:22:30 +0200 (CEST) Subject: [pypy-svn] r73853 - pypy/trunk/pypy/interpreter Message-ID: <20100417222230.8601D282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 00:22:28 2010 New Revision: 73853 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: wrap long lines Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun Apr 18 00:22:28 2010 @@ -253,7 +253,8 @@ meth = getattr(self, opdesc.methodname) res = meth(oparg, next_instr) if opdesc.index == self.opcodedesc.CALL_FUNCTION.index: - rstack.resume_point("dispatch_call", self, co_code, next_instr, ec) + rstack.resume_point("dispatch_call", self, co_code, + next_instr, ec) # !! warning, for the annotator the next line is not # comparing an int and None - you can't do that. # Instead, it's constant-folded to either True or False @@ -268,14 +269,18 @@ try: meth = getattr(self, methodname) except AttributeError: - raise BytecodeCorruption("unimplemented opcode, ofs=%d, code=%d, name=%s" % - (self.last_instr, opcode, methodname)) + raise BytecodeCorruption("unimplemented opcode, ofs=%d, " + "code=%d, name=%s" % + (self.last_instr, opcode, + methodname)) try: res = meth(oparg, next_instr) except Exception: if 0: import dis, sys - print "*** %s at offset %d (%s)" % (sys.exc_info()[0], self.last_instr, methodname) + print "*** %s at offset %d (%s)" % (sys.exc_info()[0], + self.last_instr, + methodname) try: dis.dis(co_code) except: From benjamin at codespeak.net Sun Apr 18 01:15:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:15:46 +0200 (CEST) Subject: [pypy-svn] r73854 - in pypy/trunk/pypy: objspace/flow rpython translator Message-ID: <20100417231546.C90D6282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:15:45 2010 New Revision: 73854 Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py pypy/trunk/pypy/objspace/flow/model.py pypy/trunk/pypy/objspace/flow/objspace.py pypy/trunk/pypy/objspace/flow/operation.py pypy/trunk/pypy/rpython/rint.py pypy/trunk/pypy/translator/simplify.py Log: move things around in flowspace, to improve dependencies It also locates more associated functionality in appropiate modules. Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Sun Apr 18 01:15:45 2010 @@ -3,18 +3,13 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import pyframe from pypy.interpreter.argument import ArgumentsForTranslation +from pypy.objspace.flow import operation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState from pypy.rlib import jit from pypy.tool.stdlib_opcode import host_bytecode_spec import sys -class OperationThatShouldNotBePropagatedError(OperationError): - pass - -class ImplicitOperationError(OperationError): - pass - class StopFlowing(Exception): pass @@ -271,13 +266,13 @@ self.crnt_frame = None self.topframeref = old_frameref - except OperationThatShouldNotBePropagatedError, e: + except operation.OperationThatShouldNotBePropagatedError, e: raise Exception( 'found an operation that always raises %s: %s' % ( self.space.unwrap(e.w_type).__name__, self.space.unwrap(e.get_w_value(self.space)))) - except ImplicitOperationError, e: + except operation.ImplicitOperationError, e: if isinstance(e.w_type, Constant): exc_cls = e.w_type.value else: @@ -380,7 +375,7 @@ def sys_exc_info(self): operr = ExecutionContext.sys_exc_info(self) - if isinstance(operr, ImplicitOperationError): + if isinstance(operr, operation.ImplicitOperationError): # re-raising an implicit operation makes it an explicit one w_value = operr.get_w_value(self.space) operr = OperationError(operr.w_type, w_value) @@ -491,7 +486,7 @@ def handle_operation_error(self, ec, operr, *args, **kwds): # see test_propagate_attribute_error for why this is here - if isinstance(operr, OperationThatShouldNotBePropagatedError): + if isinstance(operr, operation.OperationThatShouldNotBePropagatedError): raise operr return pyframe.PyFrame.handle_operation_error(self, ec, operr, *args, **kwds) Modified: pypy/trunk/pypy/objspace/flow/model.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/model.py (original) +++ pypy/trunk/pypy/objspace/flow/model.py Sun Apr 18 01:15:45 2010 @@ -322,6 +322,14 @@ self.concretetype = concretetype +class UnwrapException(Exception): + """Attempted to unwrap a Variable.""" + +class WrapException(Exception): + """Attempted wrapping of a type that cannot sanely appear in flow graph or + during its construction""" + + class SpaceOperation(object): __slots__ = "opname args result offset".split() Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Sun Apr 18 01:15:45 2010 @@ -6,18 +6,10 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import pyframe from pypy.objspace.flow.model import * -from pypy.objspace.flow import flowcontext -from pypy.objspace.flow.operation import FunctionByName +from pypy.objspace.flow import flowcontext, operation from pypy.rlib.unroll import unrolling_iterable, _unroller from pypy.rlib import rstackovf -debug = 0 - -class UnwrapException(Exception): - "Attempted to unwrap a Variable." - -class WrapException(Exception): - """Attempted wrapping of a type that cannot sanely appear in flow graph or during its construction""" # method-wrappers have not enough introspection in CPython if hasattr(complex.real.__get__, 'im_self'): @@ -25,6 +17,22 @@ else: type_with_bad_introspection = type(complex.real.__get__) +# the following gives us easy access to declare more for applications: +NOT_REALLY_CONST = { + Constant(sys): { + Constant('maxint'): True, + Constant('maxunicode'): True, + Constant('api_version'): True, + Constant('exit'): True, + Constant('exc_info'): True, + Constant('getrefcount'): True, + Constant('getdefaultencoding'): True, + # this is an incomplete list of true constants. + # if we add much more, a dedicated class + # might be considered for special objects. + } + } + # ______________________________________________________________________ class FlowObjSpace(ObjSpace): """NOT_RPYTHON. @@ -32,7 +40,7 @@ the space operations that the interpreter generates when it interprets (the bytecode of) some function. """ - + full_exceptions = False do_imports_immediately = True FrameClass = flowcontext.FlowSpaceFrame @@ -41,7 +49,8 @@ import __builtin__ self.concrete_mode = 1 self.w_None = Constant(None) - self.builtin = Module(self, Constant('__builtin__'), Constant(__builtin__.__dict__)) + self.builtin = Module(self, Constant('__builtin__'), + Constant(__builtin__.__dict__)) def pick_builtin(w_globals): return self.builtin self.builtin.pick_builtin = pick_builtin @@ -301,7 +310,7 @@ def do_operation_with_implicit_exceptions(self, name, *args_w): w_result = self.do_operation(name, *args_w) - self.handle_implicit_exceptions(implicit_exceptions.get(name)) + self.handle_implicit_exceptions(operation.implicit_exceptions.get(name)) return w_result def is_true(self, w_obj): @@ -347,8 +356,8 @@ if outcome is StopIteration: raise OperationError(self.w_StopIteration, w_exc_value) elif outcome is RuntimeError: - raise flowcontext.ImplicitOperationError(Constant(RuntimeError), - w_exc_value) + raise operation.ImplicitOperationError(Constant(RuntimeError), + w_exc_value) else: return w_item @@ -404,7 +413,7 @@ # raise SomeError(x) # # as shown by test_objspace.test_raise3. - + exceptions = [Exception] # *any* exception by default if isinstance(w_callable, Constant): c = w_callable.value @@ -414,7 +423,7 @@ types.ClassType, types.TypeType)) and c.__module__ in ['__builtin__', 'exceptions']): - exceptions = implicit_exceptions.get(c, None) + exceptions = operation.implicit_exceptions.get(c, None) self.handle_implicit_exceptions(exceptions) return w_res @@ -442,8 +451,7 @@ #if outcome is not Exception: #w_exc_cls = Constant(outcome) Now done by guessexception itself #pass - raise flowcontext.ImplicitOperationError(w_exc_cls, - w_exc_value) + raise operation.ImplicitOperationError(w_exc_cls, w_exc_value) def w_KeyboardInterrupt(self): # the reason to do this is: if you interrupt the flowing of a function @@ -458,85 +466,8 @@ raise RuntimeError("the interpreter raises RuntimeError during " "flow graph construction") w_RuntimeError = prebuilt_recursion_error = property(w_RuntimeError) +operation.add_operations(FlowObjSpace) -# the following gives us easy access to declare more for applications: -NOT_REALLY_CONST = { - Constant(sys): { - Constant('maxint'): True, - Constant('maxunicode'): True, - Constant('api_version'): True, - Constant('exit'): True, - Constant('exc_info'): True, - Constant('getrefcount'): True, - Constant('getdefaultencoding'): True, - # this is an incomplete list of true constants. - # if we add much more, a dedicated class - # might be considered for special objects. - } - } - -# ______________________________________________________________________ - -op_appendices = { - OverflowError: 'ovf', - IndexError: 'idx', - KeyError: 'key', - AttributeError: 'att', - TypeError: 'typ', - ZeroDivisionError: 'zer', - ValueError: 'val', - } - -implicit_exceptions = { - int: [ValueError], # built-ins that can always raise exceptions - float: [ValueError], - chr: [ValueError], - unichr: [ValueError], - unicode: [UnicodeDecodeError], - # specifying IndexError, and KeyError beyond Exception, - # allows the annotator to be more precise, see test_reraiseAnything/KeyError in - # the annotator tests - 'getitem': [IndexError, KeyError, Exception], - 'setitem': [IndexError, KeyError, Exception], - 'delitem': [IndexError, KeyError, Exception], - 'contains': [Exception], # from an r_dict - } - -def _add_exceptions(names, exc): - for name in names.split(): - lis = implicit_exceptions.setdefault(name, []) - if exc in lis: - raise ValueError, "your list is causing duplication!" - lis.append(exc) - assert exc in op_appendices - -def _add_except_ovf(names): - # duplicate exceptions and add OverflowError - for name in names.split(): - lis = implicit_exceptions.setdefault(name, [])[:] - lis.append(OverflowError) - implicit_exceptions[name+"_ovf"] = lis - -for _name in 'getattr', 'delattr': - _add_exceptions(_name, AttributeError) -for _name in 'iter', 'coerce': - _add_exceptions(_name, TypeError) -del _name - -_add_exceptions("""div mod divmod truediv floordiv pow - inplace_div inplace_mod inplace_divmod inplace_truediv - inplace_floordiv inplace_pow""", ZeroDivisionError) -_add_exceptions("""pow inplace_pow lshift inplace_lshift rshift - inplace_rshift""", ValueError) -_add_exceptions("""truediv divmod - inplace_add inplace_sub inplace_mul inplace_truediv - inplace_floordiv inplace_div inplace_mod inplace_pow - inplace_lshift""", OverflowError) # without a _ovf version -_add_except_ovf("""neg abs add sub mul - floordiv div mod pow lshift""") # with a _ovf version -_add_exceptions("""pow""", - OverflowError) # for the float case -del _add_exceptions, _add_except_ovf def extract_cell_content(c): """Get the value contained in a CPython 'cell', as read through @@ -559,127 +490,5 @@ return x.other # crashes if the cell is actually empty except AttributeError: raise ValueError("empty cell") - -def make_op(name, symbol, arity, specialnames): - if hasattr(FlowObjSpace, name): - return # Shouldn't do it - - import __builtin__ - - op = None - skip = False - arithmetic = False - - if name.startswith('del') or name.startswith('set') or name.startswith('inplace_'): - # skip potential mutators - if debug: print "Skip", name - skip = True - elif name in ['id', 'hash', 'iter', 'userdel']: - # skip potential runtime context dependecies - if debug: print "Skip", name - skip = True - elif name in ['repr', 'str']: - rep = getattr(__builtin__, name) - def op(obj): - s = rep(obj) - if s.find("at 0x") > -1: - print >>sys.stderr, "Warning: captured address may be awkward" - return s - else: - op = FunctionByName[name] - arithmetic = (name + '_ovf') in FunctionByName - - if not op: - if not skip: - if debug: print >> sys.stderr, "XXX missing operator:", name - else: - if debug: print "Can constant-fold operation: %s" % name - - def generic_operator(self, *args_w): - assert len(args_w) == arity, name+" got the wrong number of arguments" - if op: - args = [] - for w_arg in args_w: - try: - arg = self.unwrap_for_computation(w_arg) - except UnwrapException: - break - else: - args.append(arg) - else: - # All arguments are constants: call the operator now - #print >> sys.stderr, 'Constant operation', op - try: - result = op(*args) - except: - etype, evalue, etb = sys.exc_info() - msg = "generated by a constant operation: %s%r" % ( - name, tuple(args)) - raise flowcontext.OperationThatShouldNotBePropagatedError( - self.wrap(etype), self.wrap(msg)) - else: - # don't try to constant-fold operations giving a 'long' - # result. The result is probably meant to be sent to - # an intmask(), but the 'long' constant confuses the - # annotator a lot. - if arithmetic and type(result) is long: - pass - # don't constant-fold getslice on lists, either - elif name == 'getslice' and type(result) is list: - pass - # otherwise, fine - else: - try: - return self.wrap(result) - except WrapException: - # type cannot sanely appear in flow graph, - # store operation with variable result instead - pass - - #print >> sys.stderr, 'Variable operation', name, args_w - w_result = self.do_operation_with_implicit_exceptions(name, *args_w) - return w_result - - setattr(FlowObjSpace, name, generic_operator) - -for line in ObjSpace.MethodTable: - make_op(*line) - -""" -This is just a placeholder for some code I'm checking in elsewhere. -It is provenly possible to determine constantness of certain expressions -a little later. I introduced this a bit too early, together with tieing -this to something being global, which was a bad idea. -The concept is still valid, and it can be used to force something to -be evaluated immediately because it is supposed to be a constant. -One good possible use of this is loop unrolling. -This will be found in an 'experimental' folder with some use cases. -""" - -def override(): - def getattr(self, w_obj, w_name): - # handling special things like sys - # unfortunately this will never vanish with a unique import logic :-( - if w_obj in self.not_really_const: - const_w = self.not_really_const[w_obj] - if w_name not in const_w: - return self.do_operation_with_implicit_exceptions('getattr', w_obj, w_name) - return self.regular_getattr(w_obj, w_name) - - FlowObjSpace.regular_getattr = FlowObjSpace.getattr - FlowObjSpace.getattr = getattr - - # protect us from globals write access - def setitem(self, w_obj, w_key, w_val): - ec = self.getexecutioncontext() - if not (ec and w_obj is ec.w_globals): - return self.regular_setitem(w_obj, w_key, w_val) - raise SyntaxError, "attempt to modify global attribute %r in %r" % (w_key, ec.graph.func) - - FlowObjSpace.regular_setitem = FlowObjSpace.setitem - FlowObjSpace.setitem = setitem - -override() - # ______________________________________________________________________ # End of objspace.py Modified: pypy/trunk/pypy/objspace/flow/operation.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/operation.py (original) +++ pypy/trunk/pypy/objspace/flow/operation.py Sun Apr 18 01:15:45 2010 @@ -2,10 +2,25 @@ This module defines mappings between operation names and Python's built-in functions (or type constructors) implementing them. """ + +import __builtin__ +import __future__ +import operator +import types +import sys from pypy.interpreter.baseobjspace import ObjSpace -import operator, types, __future__ +from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import compile2 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift +from pypy.objspace.flow import model + + +class OperationThatShouldNotBePropagatedError(OperationError): + pass + +class ImplicitOperationError(OperationError): + pass + FunctionByName = {} # dict {"operation_name": } OperationName = {} # dict {: "operation_name"} @@ -216,8 +231,6 @@ ] def setup(): - if not hasattr(operator, 'is_'): # Python 2.2 - Table.append(('is_', lambda x, y: x is y)) # insert all operators for line in ObjSpace.MethodTable: name = line[0] @@ -235,4 +248,184 @@ Arity[name] = line[2] assert name in FunctionByName setup() -del Table # INTERNAL ONLY, use the dicts declared at the top of the file +del Table, setup # INTERNAL ONLY, use the dicts declared at the top of the file + +op_appendices = { + OverflowError: 'ovf', + IndexError: 'idx', + KeyError: 'key', + AttributeError: 'att', + TypeError: 'typ', + ZeroDivisionError: 'zer', + ValueError: 'val', + } + +implicit_exceptions = { + int: [ValueError], # built-ins that can always raise exceptions + float: [ValueError], + chr: [ValueError], + unichr: [ValueError], + unicode: [UnicodeDecodeError], + # specifying IndexError, and KeyError beyond Exception, + # allows the annotator to be more precise, see test_reraiseAnything/KeyError in + # the annotator tests + 'getitem': [IndexError, KeyError, Exception], + 'setitem': [IndexError, KeyError, Exception], + 'delitem': [IndexError, KeyError, Exception], + 'contains': [Exception], # from an r_dict + } + +def _add_exceptions(names, exc): + for name in names.split(): + lis = implicit_exceptions.setdefault(name, []) + if exc in lis: + raise ValueError, "your list is causing duplication!" + lis.append(exc) + assert exc in op_appendices + +def _add_except_ovf(names): + # duplicate exceptions and add OverflowError + for name in names.split(): + lis = implicit_exceptions.setdefault(name, [])[:] + lis.append(OverflowError) + implicit_exceptions[name+"_ovf"] = lis + +for _name in 'getattr', 'delattr': + _add_exceptions(_name, AttributeError) +for _name in 'iter', 'coerce': + _add_exceptions(_name, TypeError) +del _name + +_add_exceptions("""div mod divmod truediv floordiv pow + inplace_div inplace_mod inplace_divmod inplace_truediv + inplace_floordiv inplace_pow""", ZeroDivisionError) +_add_exceptions("""pow inplace_pow lshift inplace_lshift rshift + inplace_rshift""", ValueError) +_add_exceptions("""truediv divmod + inplace_add inplace_sub inplace_mul inplace_truediv + inplace_floordiv inplace_div inplace_mod inplace_pow + inplace_lshift""", OverflowError) # without a _ovf version +_add_except_ovf("""neg abs add sub mul + floordiv div mod pow lshift""") # with a _ovf version +_add_exceptions("""pow""", + OverflowError) # for the float case +del _add_exceptions, _add_except_ovf + +def make_op(fs, name, symbol, arity, specialnames): + if hasattr(fs, name): + return + + op = None + skip = False + arithmetic = False + + if (name.startswith('del') or + name.startswith('set') or + name.startswith('inplace_')): + # skip potential mutators + skip = True + elif name in ('id', 'hash', 'iter', 'userdel'): + # skip potential runtime context dependecies + skip = True + elif name in ('repr', 'str'): + rep = getattr(__builtin__, name) + def op(obj): + s = rep(obj) + if "at 0x" in s: + print >>sys.stderr, "Warning: captured address may be awkward" + return s + else: + op = FunctionByName[name] + arithmetic = (name + '_ovf') in FunctionByName + + if not op and not skip: + raise ValueError("XXX missing operator: %s" % (name,)) + + def generic_operator(self, *args_w): + assert len(args_w) == arity, name + " got the wrong number of arguments" + if op: + args = [] + for w_arg in args_w: + try: + arg = self.unwrap_for_computation(w_arg) + except model.UnwrapException: + break + else: + args.append(arg) + else: + # All arguments are constants: call the operator now + try: + result = op(*args) + except: + etype, evalue, etb = sys.exc_info() + msg = "generated by a constant operation: %s%r" % ( + name, tuple(args)) + raise OperationThatShouldNotBePropagatedError( + self.wrap(etype), self.wrap(msg)) + else: + # don't try to constant-fold operations giving a 'long' + # result. The result is probably meant to be sent to + # an intmask(), but the 'long' constant confuses the + # annotator a lot. + if arithmetic and type(result) is long: + pass + # don't constant-fold getslice on lists, either + elif name == 'getslice' and type(result) is list: + pass + # otherwise, fine + else: + try: + return self.wrap(result) + except model.WrapException: + # type cannot sanely appear in flow graph, + # store operation with variable result instead + pass + w_result = self.do_operation_with_implicit_exceptions(name, *args_w) + return w_result + + setattr(fs, name, generic_operator) + + +""" +This is just a placeholder for some code I'm checking in elsewhere. +It is provenly possible to determine constantness of certain expressions +a little later. I introduced this a bit too early, together with tieing +this to something being global, which was a bad idea. +The concept is still valid, and it can be used to force something to +be evaluated immediately because it is supposed to be a constant. +One good possible use of this is loop unrolling. +This will be found in an 'experimental' folder with some use cases. +""" + +def special_overrides(fs): + def getattr(self, w_obj, w_name): + # handling special things like sys + # unfortunately this will never vanish with a unique import logic :-( + if w_obj in self.not_really_const: + const_w = self.not_really_const[w_obj] + if w_name not in const_w: + return self.do_operation_with_implicit_exceptions('getattr', + w_obj, w_name) + return self.regular_getattr(w_obj, w_name) + + fs.regular_getattr = fs.getattr + fs.getattr = getattr + + # protect us from globals write access + def setitem(self, w_obj, w_key, w_val): + ec = self.getexecutioncontext() + if not (ec and w_obj is ec.w_globals): + return self.regular_setitem(w_obj, w_key, w_val) + raise SyntaxError("attempt to modify global attribute %r in %r" + % (w_key, ec.graph.func)) + + fs.regular_setitem = fs.setitem + fs.setitem = setitem + + +def add_operations(fs): + """Add function operations to the flow space.""" + for line in ObjSpace.MethodTable: + print line + make_op(fs, *line) + special_overrides(fs) Modified: pypy/trunk/pypy/rpython/rint.py ============================================================================== --- pypy/trunk/pypy/rpython/rint.py (original) +++ pypy/trunk/pypy/rpython/rint.py Sun Apr 18 01:15:45 2010 @@ -1,7 +1,7 @@ import sys from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel -from pypy.objspace.flow.objspace import op_appendices +from pypy.objspace.flow.operation import op_appendices from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Bool, Float, \ Void, Char, UniChar, malloc, pyobjectptr, UnsignedLongLong, \ SignedLongLong, build_number, Number, cast_primitive, typeOf Modified: pypy/trunk/pypy/translator/simplify.py ============================================================================== --- pypy/trunk/pypy/translator/simplify.py (original) +++ pypy/trunk/pypy/translator/simplify.py Sun Apr 18 01:15:45 2010 @@ -110,7 +110,7 @@ Instead, it will be replaced by an OP_LSHIFT_OVF operation. """ from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift - from pypy.objspace.flow.objspace import implicit_exceptions + from pypy.objspace.flow.operation import implicit_exceptions covf = Constant(ovfcheck) covfls = Constant(ovfcheck_lshift) From benjamin at codespeak.net Sun Apr 18 01:16:09 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:16:09 +0200 (CEST) Subject: [pypy-svn] r73855 - pypy/trunk/pypy/interpreter Message-ID: <20100417231609.D32C2282BDB@codespeak.net> Author: benjamin Date: Sun Apr 18 01:16:08 2010 New Revision: 73855 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: move rstack import to module level Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun Apr 18 01:16:08 2010 @@ -11,7 +11,7 @@ from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib import jit, rstackovf +from pypy.rlib import jit, rstackovf, rstack from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, @@ -76,8 +76,6 @@ def dispatch(self, pycode, next_instr, ec): # For the sequel, force 'next_instr' to be unsigned for performance - from pypy.rlib import rstack # for resume points - next_instr = r_uint(next_instr) co_code = pycode.co_code @@ -90,8 +88,6 @@ return self.popvalue() def handle_bytecode(self, co_code, next_instr, ec): - from pypy.rlib import rstack # for resume points - try: next_instr = self.dispatch_bytecode(co_code, next_instr, ec) rstack.resume_point("handle_bytecode", self, co_code, ec, @@ -235,8 +231,6 @@ return self.jump_absolute(oparg, next_instr, ec) if we_are_translated(): - from pypy.rlib import rstack # for resume points - for opdesc in unrolling_all_opcode_descs: # static checks to skip this whole case if necessary if opdesc.bytecode_spec is not self.bytecode_spec: @@ -897,7 +891,6 @@ @jit.unroll_safe def call_function(self, oparg, w_star=None, w_starstar=None): - from pypy.rlib import rstack # for resume points from pypy.interpreter.function import is_builtin_code n_arguments = oparg & 0xff @@ -930,8 +923,6 @@ self.pushvalue(w_result) def CALL_FUNCTION(self, oparg, next_instr): - from pypy.rlib import rstack # for resume points - # XXX start of hack for performance if (oparg >> 8) & 0xff == 0: # Only positional arguments From benjamin at codespeak.net Sun Apr 18 01:17:18 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:17:18 +0200 (CEST) Subject: [pypy-svn] r73856 - pypy/trunk/pypy/objspace/flow Message-ID: <20100417231718.3F033282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:17:16 2010 New Revision: 73856 Modified: pypy/trunk/pypy/objspace/flow/objspace.py Log: passing None as default is redundant Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Sun Apr 18 01:17:16 2010 @@ -423,7 +423,7 @@ types.ClassType, types.TypeType)) and c.__module__ in ['__builtin__', 'exceptions']): - exceptions = operation.implicit_exceptions.get(c, None) + exceptions = operation.implicit_exceptions.get(c) self.handle_implicit_exceptions(exceptions) return w_res From benjamin at codespeak.net Sun Apr 18 01:19:13 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:19:13 +0200 (CEST) Subject: [pypy-svn] r73857 - pypy/trunk/pypy/interpreter Message-ID: <20100417231913.1ADB8282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:19:12 2010 New Revision: 73857 Modified: pypy/trunk/pypy/interpreter/pyframe.py Log: move rstack import Modified: pypy/trunk/pypy/interpreter/pyframe.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyframe.py (original) +++ pypy/trunk/pypy/interpreter/pyframe.py Sun Apr 18 01:19:12 2010 @@ -10,7 +10,7 @@ from pypy.rlib.objectmodel import we_are_translated, instantiate from pypy.rlib.jit import hint from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib import jit +from pypy.rlib import jit, rstack from pypy.tool import stdlib_opcode # Define some opcodes used @@ -132,7 +132,6 @@ def execute_frame(self): """Execute this frame. Main entry point to the interpreter.""" - from pypy.rlib import rstack # the following 'assert' is an annotation hint: it hides from # the annotator all methods that are defined in PyFrame but # overridden in the {,Host}FrameClass subclasses of PyFrame. From benjamin at codespeak.net Sun Apr 18 01:25:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:25:35 +0200 (CEST) Subject: [pypy-svn] r73858 - pypy/trunk/pypy/objspace/flow Message-ID: <20100417232535.1C85F282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:25:33 2010 New Revision: 73858 Modified: pypy/trunk/pypy/objspace/flow/operation.py Log: kill print Modified: pypy/trunk/pypy/objspace/flow/operation.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/operation.py (original) +++ pypy/trunk/pypy/objspace/flow/operation.py Sun Apr 18 01:25:33 2010 @@ -426,6 +426,5 @@ def add_operations(fs): """Add function operations to the flow space.""" for line in ObjSpace.MethodTable: - print line make_op(fs, *line) special_overrides(fs) From benjamin at codespeak.net Sun Apr 18 01:26:01 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:26:01 +0200 (CEST) Subject: [pypy-svn] r73859 - pypy/trunk/pypy/interpreter Message-ID: <20100417232601.2D1E7282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:25:59 2010 New Revision: 73859 Modified: pypy/trunk/pypy/interpreter/pyopcode.py Log: remove uneeded import Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Sun Apr 18 01:25:59 2010 @@ -560,9 +560,8 @@ # common case raise operror else: - from pypy.interpreter.pytraceback import check_traceback msg = "raise: arg 3 must be a traceback or None" - tb = check_traceback(space, w_traceback, msg) + tb = pytraceback.check_traceback(space, w_traceback, msg) operror.application_traceback = tb # special 3-arguments raise, no new traceback obj will be attached raise RaiseWithExplicitTraceback(operror) From benjamin at codespeak.net Sun Apr 18 01:33:55 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:33:55 +0200 (CEST) Subject: [pypy-svn] r73860 - pypy/trunk/pypy/objspace/flow Message-ID: <20100417233355.28276282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:33:53 2010 New Revision: 73860 Modified: pypy/trunk/pypy/objspace/flow/objspace.py pypy/trunk/pypy/objspace/flow/specialcase.py Log: move more imports to toplevel Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Sun Apr 18 01:33:53 2010 @@ -1,14 +1,18 @@ # ______________________________________________________________________ -import sys, operator, types +import __builtin__ +import sys +import operator +import types +from pypy.tool import error from pypy.interpreter.baseobjspace import ObjSpace, Wrappable from pypy.interpreter.pycode import PyCode, cpython_code_signature from pypy.interpreter.module import Module from pypy.interpreter.error import OperationError -from pypy.interpreter import pyframe +from pypy.interpreter import pyframe, argument from pypy.objspace.flow.model import * -from pypy.objspace.flow import flowcontext, operation +from pypy.objspace.flow import flowcontext, operation, specialcase from pypy.rlib.unroll import unrolling_iterable, _unroller -from pypy.rlib import rstackovf +from pypy.rlib import rstackovf, rarithmetic # method-wrappers have not enough introspection in CPython @@ -46,7 +50,6 @@ FrameClass = flowcontext.FlowSpaceFrame def initialize(self): - import __builtin__ self.concrete_mode = 1 self.w_None = Constant(None) self.builtin = Module(self, Constant('__builtin__'), @@ -135,9 +138,8 @@ def uint_w(self, w_obj): if isinstance(w_obj, Constant): - from pypy.rlib.rarithmetic import r_uint val = w_obj.value - if type(val) is not r_uint: + if type(val) is not rarithmetic.r_uint: raise TypeError("expected unsigned: " + repr(w_obj)) return val return self.unwrap(w_obj) @@ -201,7 +203,6 @@ def setup_executioncontext(self, ec): self.executioncontext = ec - from pypy.objspace.flow import specialcase specialcase.setup(self) def exception_match(self, w_exc_type, w_check_class): @@ -264,15 +265,15 @@ graph.defaults = func.func_defaults or () self.setup_executioncontext(ec) - from pypy.tool.error import FlowingError, format_global_error - try: ec.build_flow() - except FlowingError, a: + except error.FlowingError, a: # attach additional source info to AnnotatorError _, _, tb = sys.exc_info() - e = FlowingError(format_global_error(ec.graph, ec.crnt_offset, str(a))) - raise FlowingError, e, tb + formated = error.format_global_error(ec.graph, ec.crnt_offset, + str(a)) + e = error.FlowingError(formated) + raise error.FlowingError, e, tb checkgraph(graph) return graph @@ -375,9 +376,8 @@ w_key, w_val) def call_function(self, w_func, *args_w): - from pypy.interpreter.argument import ArgumentsForTranslation nargs = len(args_w) - args = ArgumentsForTranslation(self, list(args_w)) + args = argument.ArgumentsForTranslation(self, list(args_w)) return self.call_args(w_func, args) def call_args(self, w_callable, args): Modified: pypy/trunk/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/specialcase.py (original) +++ pypy/trunk/pypy/objspace/flow/specialcase.py Sun Apr 18 01:33:53 2010 @@ -1,5 +1,4 @@ -from pypy.objspace.flow.objspace import UnwrapException -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Constant, UnwrapException from pypy.objspace.flow.operation import OperationName, Arity from pypy.interpreter.gateway import ApplevelClass from pypy.interpreter.error import OperationError From benjamin at codespeak.net Sun Apr 18 01:51:59 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 01:51:59 +0200 (CEST) Subject: [pypy-svn] r73861 - pypy/trunk/pypy/translator Message-ID: <20100417235159.7653A282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 01:51:58 2010 New Revision: 73861 Modified: pypy/trunk/pypy/translator/simplify.py Log: make imports toplevel Modified: pypy/trunk/pypy/translator/simplify.py ============================================================================== --- pypy/trunk/pypy/translator/simplify.py (original) +++ pypy/trunk/pypy/translator/simplify.py Sun Apr 18 01:51:58 2010 @@ -6,16 +6,20 @@ """ import py -from pypy.objspace.flow.model import SpaceOperation -from pypy.objspace.flow.model import Variable, Constant, Block, Link -from pypy.objspace.flow.model import c_last_exception -from pypy.objspace.flow.model import checkgraph, traverse, mkentrymap +from pypy.objspace.flow import operation +from pypy.objspace.flow.model import (SpaceOperation, Variable, Constant, Block, + Link, c_last_exception, checkgraph, + traverse, mkentrymap) +from pypy.rlib import rarithmetic +from pypy.translator import unsimplify +from pypy.translator.backendopt import ssa from pypy.rpython.lltypesystem import lloperation, lltype from pypy.rpython.ootypesystem import ootype def get_funcobj(func): """ - Return an object which is supposed to have attributes such as graph and _callable + Return an object which is supposed to have attributes such as graph and + _callable """ if hasattr(func, '_obj'): return func._obj # lltypesystem @@ -30,12 +34,9 @@ assert False def get_graph(arg, translator): - from pypy.translator.translator import graphof if isinstance(arg, Variable): return None f = arg.value - from pypy.rpython.lltypesystem import lltype - from pypy.rpython.ootypesystem import ootype if not isinstance(f, lltype._ptr) and not isinstance(f, ootype._callable): return None funcobj = get_funcobj(f) @@ -49,7 +50,7 @@ return None try: callable = funcobj._callable - return graphof(translator, callable) + return translator._graphof(callable) except (AttributeError, KeyError, AssertionError): return None @@ -109,13 +110,11 @@ ovfcheck_lshift is special because there is no preceding operation. Instead, it will be replaced by an OP_LSHIFT_OVF operation. """ - from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift - from pypy.objspace.flow.operation import implicit_exceptions - covf = Constant(ovfcheck) - covfls = Constant(ovfcheck_lshift) + covf = Constant(rarithmetic.ovfcheck) + covfls = Constant(rarithmetic.ovfcheck_lshift) def check_syntax(opname): - exlis = implicit_exceptions.get("%s_ovf" % (opname,), []) + exlis = operation.implicit_exceptions.get("%s_ovf" % (opname,), []) if OverflowError not in exlis: raise Exception("ovfcheck in %s: Operation %s has no" " overflow variant" % (graph.name, opname)) @@ -558,8 +557,7 @@ # when for all possible incoming paths they would get twice the same # value (this is really the purpose of remove_identical_vars()). # - from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder - builder = DataFlowFamilyBuilder(graph) + builder = ssa.DataFlowFamilyBuilder(graph) variable_families = builder.get_variable_families() # vertical removal while True: if not builder.merge_identical_phi_nodes(): # horizontal removal @@ -655,8 +653,7 @@ # NB. this assumes RPythonicity: we can only iterate over something # that has a len(), and this len() cannot change as long as we are # using the iterator. - from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder - builder = DataFlowFamilyBuilder(graph) + builder = ssa.DataFlowFamilyBuilder(graph) variable_families = builder.get_variable_families() c_append = Constant('append') newlist_v = {} @@ -964,7 +961,6 @@ link.args[i] = vlist2 # - wherever the list exits the loop body, add a 'hint({fence})' - from pypy.translator.unsimplify import insert_empty_block for block in loopbody: for link in block.exits: if link.target not in loopbody: @@ -976,7 +972,7 @@ link.target in stopblocks): hints['exactlength'] = True chints = Constant(hints) - newblock = insert_empty_block(None, link) + newblock = unsimplify.insert_empty_block(None, link) index = link.args.index(vlist) vlist2 = newblock.inputargs[index] vlist3 = Variable(vlist2) From benjamin at codespeak.net Sun Apr 18 02:34:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 02:34:10 +0200 (CEST) Subject: [pypy-svn] r73862 - pypy/trunk/pypy/objspace/flow Message-ID: <20100418003410.26BCC282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 02:34:08 2010 New Revision: 73862 Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py Log: kill one more case of function imports Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Sun Apr 18 02:34:08 2010 @@ -1,14 +1,14 @@ import collections +import sys from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError -from pypy.interpreter import pyframe +from pypy.interpreter import pyframe, nestedscope from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow import operation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState from pypy.rlib import jit from pypy.tool.stdlib_opcode import host_bytecode_spec -import sys class StopFlowing(Exception): pass @@ -196,8 +196,8 @@ if closure is None: self.closure = None else: - from pypy.interpreter.nestedscope import Cell - self.closure = [Cell(Constant(value)) for value in closure] + self.closure = [nestedscope.Cell(Constant(value)) + for value in closure] frame = self.create_frame() formalargcount = code.getformalargcount() arg_list = [Variable() for i in range(formalargcount)] From benjamin at codespeak.net Sun Apr 18 02:35:44 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 02:35:44 +0200 (CEST) Subject: [pypy-svn] r73863 - pypy/trunk/pypy/rlib Message-ID: <20100418003544.B3867282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 02:35:43 2010 New Revision: 73863 Modified: pypy/trunk/pypy/rlib/nonconst.py Log: remove imports Modified: pypy/trunk/pypy/rlib/nonconst.py ============================================================================== --- pypy/trunk/pypy/rlib/nonconst.py (original) +++ pypy/trunk/pypy/rlib/nonconst.py Sun Apr 18 02:35:43 2010 @@ -3,9 +3,7 @@ """ from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rpython.lltypesystem import lltype +from pypy.objspace.flow.model import Constant class NonConstant(object): def __init__(self, _constant): @@ -22,7 +20,7 @@ class EntryNonConstant(ExtRegistryEntry): _about_ = NonConstant - + def compute_result_annotation(self, arg): if hasattr(arg, 'const'): return self.bookkeeper.immutablevalue(arg.const, False) From benjamin at codespeak.net Sun Apr 18 02:42:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 02:42:35 +0200 (CEST) Subject: [pypy-svn] r73864 - pypy/trunk/pypy/translator Message-ID: <20100418004235.371DD282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 02:42:33 2010 New Revision: 73864 Modified: pypy/trunk/pypy/translator/translator.py Log: don't use star import for model Modified: pypy/trunk/pypy/translator/translator.py ============================================================================== --- pypy/trunk/pypy/translator/translator.py (original) +++ pypy/trunk/pypy/translator/translator.py Sun Apr 18 02:42:33 2010 @@ -6,9 +6,9 @@ """ import autopath, os, sys, types, copy -from pypy.objspace.flow.model import * from pypy.translator import simplify from pypy.objspace.flow.objspace import FlowObjSpace +from pypy.objspace.flow.model import FunctionGraph, checkgraph from pypy.tool.ansi_print import ansi_log from pypy.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription From benjamin at codespeak.net Sun Apr 18 02:52:58 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 02:52:58 +0200 (CEST) Subject: [pypy-svn] r73865 - pypy/trunk/pypy/annotation Message-ID: <20100418005258.74275282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 02:52:56 2010 New Revision: 73865 Modified: pypy/trunk/pypy/annotation/annrpython.py Log: remove some completely uneeded function imports Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Sun Apr 18 02:52:56 2010 @@ -1,17 +1,18 @@ -from types import FunctionType +import sys +import types from pypy.tool.ansi_print import ansi_log, raise_nicer_exception -from pypy.annotation import model as annmodel from pypy.tool.pairtype import pair +from pypy.tool.error import (format_blocked_annotation_error, + format_someobject_error, AnnotatorError) +from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, + c_last_exception, checkgraph) +from pypy.translator import simplify, transform +from pypy.annotation import model as annmodel, signature from pypy.annotation.bookkeeper import Bookkeeper -from pypy.annotation import signature -from pypy.objspace.flow.model import Variable, Constant -from pypy.objspace.flow.model import FunctionGraph -from pypy.objspace.flow.model import c_last_exception, checkgraph import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) +log = py.log.Producer("annrpython") +py.log.setconsumer("annrpython", ansi_log) -from pypy.tool.error import format_blocked_annotation_error, format_someobject_error, AnnotatorError FAIL = object() @@ -84,7 +85,7 @@ def build_types(self, function, input_arg_types, complete_now=True): """Recursively build annotations about the specific entry point.""" - assert isinstance(function, FunctionType), "fix that!" + assert isinstance(function, types.FunctionType), "fix that!" # make input arguments and set their type inputcells = [self.typeannotation(t) for t in input_arg_types] @@ -429,10 +430,8 @@ def simplify(self, block_subset=None, extra_passes=None): # Generic simplifications - from pypy.translator import transform transform.transform_graph(self, block_subset=block_subset, extra_passes=extra_passes) - from pypy.translator import simplify if block_subset is None: graphs = self.translator.graphs else: @@ -534,7 +533,6 @@ except BlockedInference, e: if annmodel.DEBUG: - import sys self.why_not_annotated[block] = sys.exc_info() if (e.op is block.operations[-1] and @@ -611,12 +609,11 @@ candidates = [c for c in candidates if c not in covered] for link in exits: - import types in_except_block = False last_exception_var = link.last_exception # may be None for non-exception link last_exc_value_var = link.last_exc_value # may be None for non-exception link - + if isinstance(link.exitcase, (types.ClassType, type)) \ and issubclass(link.exitcase, py.builtin.BaseException): assert last_exception_var and last_exc_value_var From benjamin at codespeak.net Sun Apr 18 03:19:29 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 03:19:29 +0200 (CEST) Subject: [pypy-svn] r73866 - pypy/trunk/pypy/objspace/std Message-ID: <20100418011929.4A865282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 03:19:11 2010 New Revision: 73866 Modified: pypy/trunk/pypy/objspace/std/basestringtype.py Log: simplify basestring typedef Modified: pypy/trunk/pypy/objspace/std/basestringtype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/basestringtype.py (original) +++ pypy/trunk/pypy/objspace/std/basestringtype.py Sun Apr 18 03:19:11 2010 @@ -1,9 +1,7 @@ -from pypy.objspace.std.stdtypedef import * +from pypy.objspace.std.stdtypedef import StdTypeDef -# ____________________________________________________________ - basestring_typedef = StdTypeDef("basestring", - __doc__ = '''Type basestring cannot be instantiated; it is the base for str and unicode.''' + __doc__ = ("basestring cannot be instantiated; " + "it is the base for str and unicode.") ) - From benjamin at codespeak.net Sun Apr 18 03:28:48 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 03:28:48 +0200 (CEST) Subject: [pypy-svn] r73867 - pypy/trunk/pypy/interpreter Message-ID: <20100418012848.A35B6282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 03:28:31 2010 New Revision: 73867 Modified: pypy/trunk/pypy/interpreter/gateway.py Log: unused import Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sun Apr 18 03:28:31 2010 @@ -919,7 +919,6 @@ def build_applevel_dict(self, space): "NOT_RPYTHON" - from pypy.interpreter.pycode import PyCode w_glob = space.newdict(module=True) space.setitem(w_glob, space.wrap('__name__'), space.wrap(self.modname)) space.exec_(self.source, w_glob, w_glob, From benjamin at codespeak.net Sun Apr 18 04:13:51 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 04:13:51 +0200 (CEST) Subject: [pypy-svn] r73868 - in pypy/trunk/pypy: interpreter translator/goal Message-ID: <20100418021351.77606282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 04:13:49 2010 New Revision: 73868 Removed: pypy/trunk/pypy/translator/goal/buildcache2.py Modified: pypy/trunk/pypy/interpreter/typedef.py Log: remove old script and attribute it used Modified: pypy/trunk/pypy/interpreter/typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/typedef.py (original) +++ pypy/trunk/pypy/interpreter/typedef.py Sun Apr 18 04:13:49 2010 @@ -880,6 +880,3 @@ SuspendedUnroller.typedef = TypeDef("SuspendedUnroller") SuspendedUnroller.typedef.acceptable_as_base_class = False - - -interptypes = [ val.typedef for name,val in globals().items() if hasattr(val,'__bases__') and hasattr(val,'typedef') ] From benjamin at codespeak.net Sun Apr 18 04:14:43 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 04:14:43 +0200 (CEST) Subject: [pypy-svn] r73869 - pypy/branch/interp-app-interface Message-ID: <20100418021443.4705C282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 04:14:41 2010 New Revision: 73869 Added: pypy/branch/interp-app-interface/ (props changed) - copied from r73868, pypy/trunk/ Log: branch to separate the interpreter from its applevel bindings From benjamin at codespeak.net Sun Apr 18 05:32:53 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 05:32:53 +0200 (CEST) Subject: [pypy-svn] r73870 - pypy/branch/interp-app-interface Message-ID: <20100418033253.BB315282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 05:32:52 2010 New Revision: 73870 Removed: pypy/branch/interp-app-interface/ Log: this clearly was not the way to go From benjamin at codespeak.net Sun Apr 18 05:57:01 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 05:57:01 +0200 (CEST) Subject: [pypy-svn] r73871 - pypy/trunk/pypy/interpreter Message-ID: <20100418035701.7002E282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 05:56:59 2010 New Revision: 73871 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py Log: must add functions to global table for stackless happiness Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Sun Apr 18 05:56:59 2010 @@ -240,9 +240,12 @@ identifier = self.code.identifier assert Function._all.get(identifier, self) is self, ("duplicate " "function ids") - Function._all[identifier] = self + self.add_to_table() return False + def add_to_table(self): + Function._all[self.code.identifier] = self + def find(identifier): return Function._all[identifier] find = staticmethod(find) Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sun Apr 18 05:56:59 2010 @@ -801,6 +801,8 @@ defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) + if not space.config.translating: + fn.add_to_table() if gateway.as_classmethod: fn = ClassMethod(space.wrap(fn)) return fn From benjamin at codespeak.net Sun Apr 18 06:54:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 06:54:35 +0200 (CEST) Subject: [pypy-svn] r73872 - pypy/trunk/pypy/module/_codecs Message-ID: <20100418045435.B6AB9282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 06:54:33 2010 New Revision: 73872 Modified: pypy/trunk/pypy/module/_codecs/__init__.py pypy/trunk/pypy/module/_codecs/app_codecs.py pypy/trunk/pypy/module/_codecs/interp_codecs.py Log: move some app things around to get _codecs to initialize properly Modified: pypy/trunk/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/__init__.py (original) +++ pypy/trunk/pypy/module/_codecs/__init__.py Sun Apr 18 06:54:33 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.rlib import runicode +from pypy.module._codecs import interp_codecs class Module(MixedModule): appleveldefs = { @@ -16,7 +17,6 @@ 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', - '_register_existing_errors': 'app_codecs._register_existing_errors', 'charmap_build' : 'app_codecs.charmap_build' } interpleveldefs = { @@ -57,9 +57,5 @@ MixedModule.__init__(self, space, *args) - def setup_after_space_initialization(self): - "NOT_RPYTHON" - self.space.appexec([], """(): - import _codecs - _codecs._register_existing_errors() - """) + def startup(self, space): + interp_codecs.register_builtin_error_handlers(space) Modified: pypy/trunk/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/app_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/app_codecs.py Sun Apr 18 06:54:33 2010 @@ -214,79 +214,6 @@ res = ''.join(res) return res, len(res) -def check_exception(exc): - try: - delta = exc.end - exc.start - if delta < 0 or not isinstance(exc.object, (unicode, str)): - raise TypeError("wrong exception") - except AttributeError: - raise TypeError("wrong exception") - -def strict_errors(exc): - if isinstance(exc, Exception): - raise exc - else: - raise TypeError("codec must pass exception instance") - -def ignore_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'', exc.end - elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): - return u'', exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" - -def replace_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'?'*(exc.end-exc.start), exc.end - elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): - return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -def xmlcharrefreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - res = [] - for ch in exc.object[exc.start:exc.end]: - res += '&#' - res += str(ord(ch)) - res += ';' - return u''.join(res), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - -def backslashreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - p = [] - for c in exc.object[exc.start:exc.end]: - p += '\\' - oc = ord(c) - if (oc >= 0x00010000): - p += 'U' - p += "%.8x" % ord(c) - elif (oc >= 0x100): - p += 'u' - p += "%.4x" % ord(c) - else: - p += 'x' - p += "%.2x" % ord(c) - return u''.join(p), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - - -def _register_existing_errors(): - import _codecs - _codecs.register_error("strict", strict_errors) - _codecs.register_error("ignore", ignore_errors) - _codecs.register_error("replace", replace_errors) - _codecs.register_error("xmlcharrefreplace", xmlcharrefreplace_errors) - _codecs.register_error("backslashreplace", backslashreplace_errors) - # ---------------------------------------------------------------------- ##import sys Modified: pypy/trunk/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/interp_codecs.py Sun Apr 18 06:54:33 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel from pypy.interpreter.baseobjspace import W_Root from pypy.rlib.rstring import StringBuilder, UnicodeBuilder @@ -107,7 +107,80 @@ space.w_LookupError, "unknown encoding: %s", encoding) lookup_codec.unwrap_spec = [ObjSpace, str] - + +app_errors = applevel(""" +def check_exception(exc): + try: + delta = exc.end - exc.start + if delta < 0 or not isinstance(exc.object, (unicode, str)): + raise TypeError("wrong exception") + except AttributeError: + raise TypeError("wrong exception") + +def strict_errors(exc): + if isinstance(exc, Exception): + raise exc + else: + raise TypeError("codec must pass exception instance") + +def ignore_errors(exc): + check_exception(exc) + if isinstance(exc, UnicodeEncodeError): + return u'', exc.end + elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): + return u'', exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%exc) + +Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" + +def replace_errors(exc): + check_exception(exc) + if isinstance(exc, UnicodeEncodeError): + return u'?'*(exc.end-exc.start), exc.end + elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): + return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%exc) + +def xmlcharrefreplace_errors(exc): + if isinstance(exc, UnicodeEncodeError): + res = [] + for ch in exc.object[exc.start:exc.end]: + res += '&#' + res += str(ord(ch)) + res += ';' + return u''.join(res), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) + +def backslashreplace_errors(exc): + if isinstance(exc, UnicodeEncodeError): + p = [] + for c in exc.object[exc.start:exc.end]: + p += '\\\\' + oc = ord(c) + if (oc >= 0x00010000): + p += 'U' + p += "%.8x" % ord(c) + elif (oc >= 0x100): + p += 'u' + p += "%.4x" % ord(c) + else: + p += 'x' + p += "%.2x" % ord(c) + return u''.join(p), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) +""") + +def register_builtin_error_handlers(space): + state = space.fromcache(CodecState) + for error in ("strict", "ignore", "replace", "xmlcharrefreplace", + "backslashreplace"): + name = error + "_errors" + state.codec_error_registry[error] = app_errors.wget(space, name) + def lookup_error(space, errors): """lookup_error(errors) -> handler From benjamin at codespeak.net Sun Apr 18 07:10:09 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 07:10:09 +0200 (CEST) Subject: [pypy-svn] r73873 - pypy/trunk/pypy/module/sys Message-ID: <20100418051009.98950282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 07:10:07 2010 New Revision: 73873 Modified: pypy/trunk/pypy/module/sys/__init__.py Log: this makes test_app_main work; I have no idea why Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Sun Apr 18 07:10:07 2010 @@ -1,6 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError -from pypy.module.sys.interp_encoding import _getfilesystemencoding class Module(MixedModule): """Sys Builtin Module. """ @@ -88,6 +87,7 @@ self.space.setitem(w_modules, w_name, w_module) def startup(self, space): + from pypy.module.sys.interp_encoding import _getfilesystemencoding self.filesystemencoding = _getfilesystemencoding(space) def getmodule(self, name): From benjamin at codespeak.net Sun Apr 18 19:41:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 19:41:45 +0200 (CEST) Subject: [pypy-svn] r73874 - pypy/trunk/pypy/module/_codecs Message-ID: <20100418174145.1A4E4282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 19:41:43 2010 New Revision: 73874 Modified: pypy/trunk/pypy/module/_codecs/__init__.py Log: make builtin error handlers staticly registered Modified: pypy/trunk/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/__init__.py (original) +++ pypy/trunk/pypy/module/_codecs/__init__.py Sun Apr 18 19:41:43 2010 @@ -57,5 +57,4 @@ MixedModule.__init__(self, space, *args) - def startup(self, space): interp_codecs.register_builtin_error_handlers(space) From benjamin at codespeak.net Sun Apr 18 19:50:25 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 18 Apr 2010 19:50:25 +0200 (CEST) Subject: [pypy-svn] r73875 - pypy/trunk/pypy/module/posix/test Message-ID: <20100418175025.05C00282B90@codespeak.net> Author: benjamin Date: Sun Apr 18 19:50:24 2010 New Revision: 73875 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: handle systems with a crummy filessytem encoding by default Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Sun Apr 18 19:50:24 2010 @@ -38,10 +38,12 @@ if os.name == 'nt': py.test.skip("no sparse files on Windows") +GET_POSIX = "(): import %s as m ; return m" % os.name + class AppTestPosix: def setup_class(cls): cls.space = space - cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) + cls.w_posix = space.appexec([], GET_POSIX) cls.w_path = space.wrap(str(path)) cls.w_path2 = space.wrap(str(path2)) cls.w_pdir = space.wrap(str(pdir)) @@ -143,13 +145,6 @@ st = self.posix.lstat(".") assert stat.S_ISDIR(st.st_mode) - def test_stat_unicode(self): - # test that passing unicode would not raise UnicodeDecodeError - try: - self.posix.stat(u"?") - except OSError: - pass - def test_stat_exception(self): import sys, errno for fn in [self.posix.stat, self.posix.lstat]: @@ -647,6 +642,25 @@ assert isinstance(f, file) assert f.read() == 'xxx' +class AppTestPosixUnicode: + + def setup_class(cls): + cls.space = space + cls.w_posix = space.appexec([], GET_POSIX) + cls.save_fs_encoding = space.sys.filesystemencoding + space.sys.filesystemencoding = "utf-8" + + def teardown_class(cls): + cls.space.sys.filesystemencoding = cls.save_fs_encoding + + def test_stat_unicode(self): + # test that passing unicode would not raise UnicodeDecodeError + try: + self.posix.stat(u"?") + except OSError: + pass + + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible def setup_class(cls): From agaynor at codespeak.net Sun Apr 18 21:33:15 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 18 Apr 2010 21:33:15 +0200 (CEST) Subject: [pypy-svn] r73876 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20100418193315.6E3C4282B90@codespeak.net> Author: agaynor Date: Sun Apr 18 21:33:12 2010 New Revision: 73876 Modified: pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: os.open should use the file system encoding. Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Sun Apr 18 21:33:12 2010 @@ -20,7 +20,7 @@ except OSError, e: raise wrap_oserror(space, e, fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, str, "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Sun Apr 18 21:33:12 2010 @@ -659,6 +659,13 @@ self.posix.stat(u"?") except OSError: pass + + def test_open_unicode(self): + # Ensure passing unicode doesn't raise UnicodeEncodeError + try: + self.posix.open(u"?", self.posix.O_WRONLY) + except OSError: + pass class TestPexpect(object): From afa at codespeak.net Sun Apr 18 22:18:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 18 Apr 2010 22:18:19 +0200 (CEST) Subject: [pypy-svn] r73877 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100418201819.37D58282B90@codespeak.net> Author: afa Date: Sun Apr 18 22:18:17 2010 New Revision: 73877 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Move type initialization into typeobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Sun Apr 18 22:18:17 2010 @@ -434,55 +434,10 @@ BOOTSTRAP_FUNCTIONS.append(func) return func -def bootstrap_types(space): - from pypy.module.cpyext.pyobject import make_ref, create_ref, track_reference - from pypy.module.cpyext.typeobject import PyTypeObjectPtr - +def run_bootstrap_functions(space): for func in BOOTSTRAP_FUNCTIONS: func(space) - # some types are difficult to create because of cycles. - # - object.ob_type = type - # - type.ob_type = type - # - tuple.ob_type = type - # - type.tp_base = object - # - tuple.tp_base = object - # - type.tp_bases is a tuple - # - object.tp_bases is a tuple - # - tuple.tp_bases is a tuple - - # insert null placeholders to please make_ref() - state = space.fromcache(State) - state.py_objects_w2r[space.w_type] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_object] = lltype.nullptr(PyObject.TO) - state.py_objects_w2r[space.w_tuple] = lltype.nullptr(PyObject.TO) - - # create the objects - py_type = create_ref(space, space.w_type) - py_object = create_ref(space, space.w_object) - py_tuple = create_ref(space, space.w_tuple) - - # form cycles - pto_type = rffi.cast(PyTypeObjectPtr, py_type) - py_type.c_ob_type = pto_type - py_object.c_ob_type = pto_type - py_tuple.c_ob_type = pto_type - - pto_object = rffi.cast(PyTypeObjectPtr, py_object) - pto_type.c_tp_base = pto_object - pto_tuple = rffi.cast(PyTypeObjectPtr, py_tuple) - pto_tuple.c_tp_base = pto_object - - pto_type.c_tp_bases.c_ob_type = pto_tuple - pto_object.c_tp_bases.c_ob_type = pto_tuple - pto_tuple.c_tp_bases.c_ob_type = pto_tuple - - # Restore the mapping - track_reference(space, py_type, space.w_type) - track_reference(space, py_object, space.w_object) - track_reference(space, py_tuple, space.w_tuple) - - #_____________________________________________________ # Build the bridge DLL, Allow extension DLLs to call # back into Pypy space functions @@ -530,7 +485,7 @@ outputfilename=str(udir / "module_cache" / "pypyapi")) modulename = py.path.local(eci.libraries[-1]) - bootstrap_types(space) + run_bootstrap_functions(space) # load the bridge, and init structure import ctypes @@ -686,7 +641,7 @@ eci = build_eci(False, export_symbols, code) - bootstrap_types(space) + run_bootstrap_functions(space) setup_va_functions(eci) # populate static data Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Sun Apr 18 22:18:17 2010 @@ -17,7 +17,8 @@ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ CANNOT_FAIL, PyBufferProcs -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, get_typedescr, make_typedescr +from pypy.module.cpyext.pyobject import PyObject, make_ref, create_ref, from_ref +from pypy.module.cpyext.pyobject import get_typedescr, make_typedescr, track_reference from pypy.interpreter.module import Module from pypy.interpreter.function import FunctionWithFixedCode, StaticMethod from pypy.module.cpyext import structmemberdefs @@ -332,6 +333,48 @@ realize=type_realize, dealloc=type_dealloc) + # some types are difficult to create because of cycles. + # - object.ob_type = type + # - type.ob_type = type + # - tuple.ob_type = type + # - type.tp_base = object + # - tuple.tp_base = object + # - type.tp_bases is a tuple + # - object.tp_bases is a tuple + # - tuple.tp_bases is a tuple + + # insert null placeholders to please make_ref() + state = space.fromcache(State) + state.py_objects_w2r[space.w_type] = lltype.nullptr(PyObject.TO) + state.py_objects_w2r[space.w_object] = lltype.nullptr(PyObject.TO) + state.py_objects_w2r[space.w_tuple] = lltype.nullptr(PyObject.TO) + + # create the objects + py_type = create_ref(space, space.w_type) + py_object = create_ref(space, space.w_object) + py_tuple = create_ref(space, space.w_tuple) + + # form cycles + pto_type = rffi.cast(PyTypeObjectPtr, py_type) + py_type.c_ob_type = pto_type + py_object.c_ob_type = pto_type + py_tuple.c_ob_type = pto_type + + pto_object = rffi.cast(PyTypeObjectPtr, py_object) + pto_type.c_tp_base = pto_object + pto_tuple = rffi.cast(PyTypeObjectPtr, py_tuple) + pto_tuple.c_tp_base = pto_object + + pto_type.c_tp_bases.c_ob_type = pto_tuple + pto_object.c_tp_bases.c_ob_type = pto_tuple + pto_tuple.c_tp_bases.c_ob_type = pto_tuple + + # Restore the mapping + track_reference(space, py_type, space.w_type) + track_reference(space, py_object, space.w_object) + track_reference(space, py_tuple, space.w_tuple) + + @cpython_api([PyObject], lltype.Void, external=False) def subtype_dealloc(space, obj): pto = obj.c_ob_type From afa at codespeak.net Mon Apr 19 01:11:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Apr 2010 01:11:48 +0200 (CEST) Subject: [pypy-svn] r73878 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100418231148.A487C282B90@codespeak.net> Author: afa Date: Mon Apr 19 01:11:46 2010 New Revision: 73878 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: kill a XXX Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 19 01:11:46 2010 @@ -233,7 +233,7 @@ INTERPLEVEL_API = {} FUNCTIONS = {} FUNCTIONS_STATIC = {} -FUNCTIONS_C = [ # XXX rename to SYMBOLS_C +SYMBOLS_C = [ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', @@ -445,7 +445,7 @@ def build_bridge(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) + export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -630,7 +630,7 @@ def setup_library(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + FUNCTIONS_C + list(GLOBALS) + export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() From agaynor at codespeak.net Mon Apr 19 03:48:16 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 19 Apr 2010 03:48:16 +0200 (CEST) Subject: [pypy-svn] r73879 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20100419014816.A8C6E282B90@codespeak.net> Author: agaynor Date: Mon Apr 19 03:48:14 2010 New Revision: 73879 Modified: pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: Fix os.unlink and os.remove, again. Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Mon Apr 19 03:48:14 2010 @@ -284,7 +284,7 @@ os.unlink(path) except OSError, e: raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, str] +unlink.unwrap_spec = [ObjSpace, 'path'] def remove(space, path): """Remove a file (same as unlink(path)).""" @@ -292,7 +292,7 @@ os.unlink(path) except OSError, e: raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, str] +remove.unwrap_spec = [ObjSpace, 'path'] def _getfullpathname(space, path): """helper for ntpath.abspath """ Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Mon Apr 19 03:48:14 2010 @@ -666,6 +666,13 @@ self.posix.open(u"?", self.posix.O_WRONLY) except OSError: pass + + def test_remove_unicode(self): + # See 2 above ;) + try: + self.posix.remove(u"?") + except OSError: + pass class TestPexpect(object): From benjamin at codespeak.net Mon Apr 19 04:19:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 19 Apr 2010 04:19:46 +0200 (CEST) Subject: [pypy-svn] r73880 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20100419021946.7FB08282B90@codespeak.net> Author: benjamin Date: Mon Apr 19 04:19:45 2010 New Revision: 73880 Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py Log: correct error Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/metaparser.py Mon Apr 19 04:19:45 2010 @@ -230,7 +230,8 @@ for label, their_first in overlap_check.iteritems(): for sub_label in their_first: if sub_label in inverse: - raise PgenError("ambiguous symbol %s" % (symbol,)) + raise PgenError("ambiguous symbol with label %s" + % (label,)) inverse[sub_label] = label self.first[name] = all_labels return all_labels From arigo at codespeak.net Mon Apr 19 10:51:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Apr 2010 10:51:03 +0200 (CEST) Subject: [pypy-svn] r73881 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100419085103.C5EEA282B9C@codespeak.net> Author: arigo Date: Mon Apr 19 10:51:01 2010 New Revision: 73881 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/ (props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/__init__.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py - copied unchanged from r73730, pypy/branch/blackhole-improvement/pypy/jit/metainterp/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/ (props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/__init__.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (contents, props changed) Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter2.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/support.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter2.py Log: Start splitting codewriter.py into multiple files that live in their own directory. Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/__init__.py ============================================================================== Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,169 @@ +from pypy.objspace.flow.model import Variable + + +class SSARepr(object): + def __init__(self): + self.insns = [] + +class Label(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return "Label(%r)" % (self.name, ) + def __eq__(self, other): + return isinstance(other, Label) and other.name == self.name + +class TLabel(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return "TLabel(%r)" % (self.name, ) + def __eq__(self, other): + return isinstance(other, TLabel) and other.name == self.name + +class Register(object): + def __init__(self, index): + self.index = index + +# ____________________________________________________________ + +def flatten_graph(graph, regalloc): + """Flatten the graph into an SSARepr, with already-computed register + allocations.""" + flattener = GraphFlattener(graph, regalloc) + flattener.enforce_input_args() + flattener.generate_ssa_form() + return flattener.assembler + + +class GraphFlattener(object): + + def __init__(self, graph, regalloc): + self.graph = graph + self.regalloc = regalloc + self.registers = {} + + def enforce_input_args(self): + inputargs = self.graph.startblock.inputargs + for i in range(len(inputargs)): + col = self.regalloc.getcolor(inputargs[i]) + if col != i: + assert col > i + self.regalloc.swapcolors(i, col) + for i in range(len(inputargs)): + assert self.regalloc.getcolor(inputargs[i]) == i + + def generate_ssa_form(self): + self.assembler = SSARepr() + self.seen_blocks = {} + self.make_bytecode_block(self.graph.startblock) + + def make_bytecode_block(self, block): + if block.exits == (): + self.make_return(block.inputargs) + return + if block in self.seen_blocks: + self.emitline("goto", TLabel(block)) + return + # inserting a goto not necessary, falling through + self.seen_blocks[block] = True + self.emitline(Label(block)) + # + operations = block.operations + for i, op in enumerate(operations): + self.serialize_op(op) + # + self.insert_exits(block) + + def make_return(self, args): + if len(args) == 1: + # return from function + self.emitline("int_return", self.getcolor(args[0])) + elif len(args) == 2: + # exception block, raising an exception from a function + xxx + else: + raise Exception("?") + + def make_link(self, link): + if link.target.exits == (): + self.make_return(link.args) + return + self.insert_renamings(link) + self.make_bytecode_block(link.target) + + def insert_exits(self, block): + if len(block.exits) == 1: + link = block.exits[0] + assert link.exitcase is None + self.make_link(link) + else: + assert len(block.exits) == 2 + linkfalse, linktrue = block.exits + if linkfalse.llexitcase == True: + linkfalse, linktrue = linktrue, linkfalse + # + self.emitline('goto_if_not', TLabel(linkfalse), + self.getcolor(block.exitswitch)) + # true path: + self.make_link(linktrue) + # false path: + self.emitline(Label(linkfalse)) + self.make_link(linkfalse) + + def optimize_goto_if_not(self, block): + xxxxxxx + if not self.optimize: + raise CannotOptimize + v = block.exitswitch + for link in block.exits: + if v in link.args: + raise CannotOptimize # variable escapes to next block + for op in block.operations[::-1]: + if v in op.args: + raise CannotOptimize # variable is also used in cur block + if v is op.result: + if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', + 'int_gt', 'int_ge'): + raise CannotOptimize # not a supported operation + killop = (op.opname,) + tuple(op.args) + (v,) + self.assembler.insns.remove(killop) + return 'goto_if_not_' + op.opname, op.args + raise CannotOptimize # variable is not produced in cur block + + def insert_renamings(self, link): + renamings_from = [] + renamings_to = [] + lst = [(v, self.getcolor(link.target.inputargs[i])) + for i, v in enumerate(link.args)] + lst.sort(key=lambda(v, w): w.index) + for v, w in lst: + if isinstance(v, Variable): + v = self.getcolor(v) + if v == w: + continue + renamings_from.append(v) + renamings_to.append(w) + if renamings_from: + self.emitline('int_rename', renamings_from, renamings_to) + + def emitline(self, *line): + self.assembler.insns.append(line) + + def serialize_op(self, op): + args = [] + for v in op.args: + if isinstance(v, Variable): + v = self.getcolor(v) + args.append(v) + if op.result is not None: + args.append(self.getcolor(op.result)) + self.emitline(op.opname, *args) + + def getcolor(self, v): + col = self.regalloc.getcolor(v) + try: + r = self.registers[col] + except KeyError: + r = self.registers[col] = Register(col) + return r Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,46 @@ +import py +from pypy.objspace.flow.model import Constant +from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register + + +def format_assembler(ssarepr): + """For testing: format a SSARepr as a multiline string.""" + from cStringIO import StringIO + seen = {} + # + def repr(x): + if isinstance(x, Register): + return '%i' + str(x.index) + elif isinstance(x, Constant): + return '$' + str(x.value) + elif isinstance(x, TLabel): + return getlabelname(x) + elif isinstance(x, list): + return '[%s]' % ', '.join(map(repr, x)) + else: + return `x` # ? + # + seenlabels = {} + for asm in ssarepr.insns: + for x in asm: + if isinstance(x, TLabel): + seenlabels[x.name] = -1 + labelcount = [0] + def getlabelname(lbl): + if seenlabels[lbl.name] == -1: + labelcount[0] += 1 + seenlabels[lbl.name] = labelcount[0] + return 'L%d' % seenlabels[lbl.name] + # + output = StringIO() + for asm in ssarepr.insns: + if isinstance(asm[0], Label): + if asm[0].name in seenlabels: + print >> output, '%s:' % getlabelname(asm[0]) + else: + print >> output, asm[0], + if len(asm) > 1: + print >> output, ', '.join(map(repr, asm[1:])) + else: + print >> output + return output.getvalue() Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,100 @@ +import sys +from pypy.objspace.flow.model import Variable +from pypy.tool.algo.color import DependencyGraph +from pypy.tool.algo.unionfind import UnionFind + + +def perform_register_allocation(graph): + regalloc = RegAllocator(graph) + regalloc.make_dependencies() + regalloc.coalesce_variables() + regalloc.find_node_coloring() + return regalloc + + +class RegAllocator(object): + DEBUG_REGALLOC = False + + def __init__(self, graph): + self.graph = graph + + def make_dependencies(self): + dg = DependencyGraph() + for block in self.graph.iterblocks(): + # Compute die_at = {Variable: index_of_operation_with_last_usage} + die_at = dict.fromkeys(block.inputargs, 0) + for i, op in enumerate(block.operations): + for v in op.args: + if isinstance(v, Variable): + die_at[v] = i + if op.result is not None: + die_at[op.result] = i + die_at.pop(block.exitswitch, None) + for link in block.exits: + for v in link.args: + die_at.pop(v, None) + # Add the variables of this block to the dependency graph + for i, v in enumerate(block.inputargs): + dg.add_node(v) + for j in range(i): + dg.add_edge(block.inputargs[j], v) + livevars = set(block.inputargs) + die_at = [(value, key) for (key, value) in die_at.items()] + die_at.sort() + die_at.append((sys.maxint,)) + die_index = 0 + for i, op in enumerate(block.operations): + while die_at[die_index][0] == i: + livevars.remove(die_at[die_index][1]) + die_index += 1 + if op.result is not None: + livevars.add(op.result) + dg.add_node(op.result) + for v in livevars: + dg.add_edge(v, op.result) + self._depgraph = dg + + def coalesce_variables(self): + uf = UnionFind() + dg = self._depgraph + pendingblocks = list(self.graph.iterblocks()) + while pendingblocks: + block = pendingblocks.pop() + # Aggressively try to coalesce each source variable with its + # target. We start from the end of the graph instead of + # from the beginning. This is a bit arbitrary, but the idea + # is that the end of the graph runs typically more often + # than the start, given that we resume execution from the + # middle during blackholing. + for link in block.exits: + for i, v in enumerate(link.args): + if isinstance(v, Variable): + w = link.target.inputargs[i] + v0 = uf.find_rep(v) + w0 = uf.find_rep(w) + if v0 is not w0 and v0 not in dg.neighbours[w0]: + _, rep, _ = uf.union(v0, w0) + if rep is v0: + dg.coalesce(w0, v0) + else: + assert rep is w0 + dg.coalesce(v0, w0) + self._unionfind = uf + + def find_node_coloring(self): + self._coloring = self._depgraph.find_node_coloring() + if self.DEBUG_REGALLOC: + for block in self.graph.iterblocks(): + print block + for v in block.getvariables(): + print '\t', v, '\t', self.getcolor(v) + + def getcolor(self, v): + return self._coloring[self._unionfind.find_rep(v)] + + def swapcolors(self, col1, col2): + for key, value in self._coloring.items(): + if value == col1: + self._coloring[key] = col2 + elif value == col2: + self._coloring[key] = col1 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/__init__.py ============================================================================== Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,58 @@ +import py +from pypy.jit.codewriter import support +from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.format import format_assembler + + +class FakeRegAlloc: + # a RegAllocator that answers "0, 1, 2, 3, 4..." for the colors + def __init__(self): + self.seen = {} + self.num_colors = 0 + def getcolor(self, v): + if v not in self.seen: + self.seen[v] = self.num_colors + self.num_colors += 1 + return self.seen[v] + + +class TestFlatten: + + def make_graphs(self, func, values, type_system='lltype'): + self.rtyper = support.annotate(func, values, type_system=type_system) + return self.rtyper.annotator.translator.graphs + + def encoding_test(self, func, args, expected, optimize=True): + graphs = self.make_graphs(func, args) + ssarepr = flatten_graph(graphs[0], FakeRegAlloc()) + asm = format_assembler(ssarepr) + expected = str(py.code.Source(expected)).strip() + '\n' + assert asm == expected + + def test_simple(self): + def f(n): + return n + 10 + self.encoding_test(f, [5], """ + int_add %i0, $10, %i1 + int_return %i1 + """) + + def test_loop(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + self.encoding_test(f, [5, 6], """ + int_rename [%i0, %i1], [%i2, %i3] + L1: + int_gt %i2, $0, %i4 + goto_if_not L2, %i4 + int_rename [%i2, %i3], [%i5, %i6] + int_add %i6, %i5, %i7 + int_sub %i5, $1, %i8 + int_rename [%i8, %i7], [%i2, %i3] + goto L1 + L2: + int_return %i3 + """) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,47 @@ +import py +from pypy.objspace.flow.model import Constant +from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.flatten import Label, TLabel, SSARepr, Register + + +def test_format_assembler_simple(): + ssarepr = SSARepr() + i0, i1, i2 = Register(0), Register(1), Register(2) + ssarepr.insns = [ + ('livevars', [i0, i1]), + ('int_add', i0, i1, i2), + ('int_return', i2), + ] + asm = format_assembler(ssarepr) + expected = """ + livevars [%i0, %i1] + int_add %i0, %i1, %i2 + int_return %i2 + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' + +def test_format_assembler_loop(): + ssarepr = SSARepr() + i0, i1 = Register(0), Register(1) + ssarepr.insns = [ + ('livevars', [i0, i1]), + (Label('L1'),), + ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0)), + ('int_add', i1, i0, i1), + ('int_sub', i0, Constant(1), i0), + ('goto', TLabel('L1')), + (Label('L2'),), + ('int_return', i1), + ] + asm = format_assembler(ssarepr) + expected = """ + livevars [%i0, %i1] + L1: + goto_if_not_int_gt L2, %i0, $0 + int_add %i1, %i0, %i1 + int_sub %i0, $1, %i0 + goto L1 + L2: + int_return %i1 + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Mon Apr 19 10:51:01 2010 @@ -0,0 +1,111 @@ +import py +from pypy.jit.codewriter import support +from pypy.jit.codewriter.regalloc import perform_register_allocation +from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.format import format_assembler + + +class TestRegAlloc: + + def make_graphs(self, func, values, type_system='lltype'): + self.rtyper = support.annotate(func, values, type_system=type_system) + return self.rtyper.annotator.translator.graphs + + def check_assembler(self, graph, expected): + regalloc = perform_register_allocation(graph) + ssarepr = flatten_graph(graph, regalloc) + asm = format_assembler(ssarepr) + assert asm == str(py.code.Source(expected).strip()) + '\n' + + def test_regalloc_simple(self): + def f(a, b): + return a + b + graph = self.make_graphs(f, [5, 6])[0] + regalloc = perform_register_allocation(graph) + va, vb = graph.startblock.inputargs + vc = graph.startblock.operations[0].result + assert regalloc.getcolor(va) == 0 + assert regalloc.getcolor(vb) == 1 + assert regalloc.getcolor(vc) == 0 + + def test_regalloc_loop(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + graph = self.make_graphs(f, [5, 6])[0] + self.check_assembler(graph, """ + L1: + int_gt %i0, $0, %i2 + goto_if_not L2, %i2 + int_add %i1, %i0, %i1 + int_sub %i0, $1, %i0 + goto L1 + L2: + int_return %i1 + """) + + def test_regalloc_loop_swap(self): + def f(a, b): + while a > 0: + a, b = b, a + return b + graph = self.make_graphs(f, [5, 6])[0] + self.check_assembler(graph, """ + L1: + int_gt %i0, $0, %i2 + goto_if_not L2, %i2 + int_rename [%i1, %i0], [%i0, %i1] + goto L1 + L2: + int_return %i1 + """) + + def test_regalloc_loop_constant(self): + def f(a, b): + while a > 0: + a, b = b, 2 + return b + graph = self.make_graphs(f, [5, 6])[0] + self.check_assembler(graph, """ + L1: + int_gt %i0, $0, %i0 + goto_if_not L2, %i0 + int_rename [%i1, $2], [%i0, %i1] + goto L1 + L2: + int_return %i1 + """) + + def test_regalloc_cycle(self): + def f(a, b, c): + while a > 0: + a, b, c = b, c, a + return b + graph = self.make_graphs(f, [5, 6, 7])[0] + self.check_assembler(graph, """ + L1: + int_gt %i0, $0, %i3 + goto_if_not L2, %i3 + int_rename [%i1, %i2, %i0], [%i0, %i1, %i2] + goto L1 + L2: + int_return %i1 + """) + + def test_regalloc_same_as_var(self): + def f(a, b, c): + while a > 0: + b = c + return b + graph = self.make_graphs(f, [5, 6, 7])[0] + self.check_assembler(graph, """ + L1: + int_gt %i0, $0, %i3 + goto_if_not L2, %i3 + int_rename [%i2], [%i1] + goto L1 + L2: + int_return %i1 + """) From arigo at codespeak.net Mon Apr 19 10:51:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Apr 2010 10:51:17 +0200 (CEST) Subject: [pypy-svn] r73882 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100419085117.4D1E9282BAD@codespeak.net> Author: arigo Date: Mon Apr 19 10:51:15 2010 New Revision: 73882 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py Log: Remove unneeded attribute. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Mon Apr 19 10:51:15 2010 @@ -394,7 +394,6 @@ self.assembler = [] self.constants = [] self.positions = {} - self.blocks = {} self.seen_blocks = {} self.dont_minimize_variables = 0 self.pending_exception_handlers = [] From afa at codespeak.net Mon Apr 19 15:52:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Apr 2010 15:52:00 +0200 (CEST) Subject: [pypy-svn] r73883 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100419135200.5253A282B9E@codespeak.net> Author: afa Date: Mon Apr 19 15:51:58 2010 New Revision: 73883 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/import_.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Log: Fix PyImport_Import: cpython also builds a custom "globals" dictionary with the standard builtin when no frame is available. Also improve error messages when a function raises an error Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 19 15:51:58 2010 @@ -200,7 +200,10 @@ state = space.fromcache(State) e.normalize_exception(space) state.set_exception(e.w_type, e.get_w_value(space)) - return api_function.error_value + if restype is PyObject: + return None + else: + return api_function.error_value finally: if api_function.borrowed: state = space.fromcache(State) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/import_.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/import_.py Mon Apr 19 15:51:58 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import module from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject +from pypy.rpython.lltypesystem import rffi from pypy.interpreter.error import OperationError @cpython_api([PyObject], PyObject) @@ -12,18 +13,28 @@ Always uses absolute imports.""" caller = space.getexecutioncontext().gettopframe_nohidden() + # Get the builtins from current globals if caller is not None: w_globals = caller.w_globals - try: - w_builtin = space.getitem(w_globals, space.wrap('__builtins__')) - except OperationError, e: - if not e.match(space, space.w_KeyError): - raise - else: - if space.is_true(space.isinstance(w_builtin, space.w_dict)): - w_builtin = module.Module(space, None, w_builtin) - builtin = space.interpclass_w(w_builtin) - if isinstance(builtin, module.Module): - return space.call(builtin.get("__import__"), space.newtuple([w_name])) - raise OperationError(space.w_KeyError, space.wrap("__builtins__")) + w_builtin = space.getitem(w_globals, space.wrap('__builtins__')) + else: + # No globals -- use standard builtins, and fake globals + w_builtin = space.getbuiltinmodule('__builtin__') + w_globals = space.newdict() + space.setitem(w_globals, space.wrap("__builtins__"), w_builtin) + # Get the __import__ function from the builtins + if space.is_true(space.isinstance(w_builtin, space.w_dict)): + w_import = space.getitem(w_builtin, space.wrap("__import__")) + else: + w_import = space.getattr(w_builtin, space.wrap("__import__")) + + # Call the __import__ function with the proper argument list + # Always use absolute import here. + return space.call(w_import, space.newtuple( + [w_name, w_globals, w_globals, + space.newlist([space.wrap("__doc__")])])) + + at cpython_api([rffi.CCHARP], PyObject) +def PyImport_ImportModule(space, name): + return PyImport_Import(space, space.wrap(name)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py Mon Apr 19 15:51:58 2010 @@ -42,7 +42,11 @@ def teardown_method(self, func): state = self.space.fromcache(State) - assert state.exc_value is None + try: + state.check_and_raise_exception() + except OperationError, e: + print e.errorstr(self.space) + raise if check_and_print_leaks(self): assert False, "Test leaks or loses object(s)." Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Mon Apr 19 15:51:58 2010 @@ -2,12 +2,16 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestImport(BaseApiTest): + def setup_method(self, func): + from pypy.module.imp.importing import importhook + importhook(self.space, "os") # warm up reference counts + BaseApiTest.setup_method(self, func) + def test_import(self, space, api): # failing because we dont have a caller - skip("Fails currently, dont know how to fix") pdb = api.PyImport_Import(space.wrap("pdb")) assert pdb - assert pdb.get("pm") + assert space.getattr(pdb, space.wrap("pm")) class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): From afa at codespeak.net Mon Apr 19 16:00:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Apr 2010 16:00:58 +0200 (CEST) Subject: [pypy-svn] r73884 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100419140058.B3302282B9E@codespeak.net> Author: afa Date: Mon Apr 19 16:00:57 2010 New Revision: 73884 Added: pypy/branch/cpython-extension/pypy/module/cpyext/datetime.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_datetime.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Start implementing datetime.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Mon Apr 19 16:00:57 2010 @@ -64,6 +64,7 @@ import pypy.module.cpyext.sliceobject import pypy.module.cpyext.stubsactive import pypy.module.cpyext.pystate +import pypy.module.cpyext.datetime # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/datetime.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/datetime.py Mon Apr 19 16:00:57 2010 @@ -0,0 +1,16 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.api import cpython_api +from pypy.module.cpyext.import_ import PyImport_Import + + at cpython_api([], lltype.Void) +def PyDateTime_IMPORT(space): + return + + at cpython_api([rffi.INT, rffi.INT, rffi.INT, rffi.INT], PyObject) +def PyTime_FromTime(space, hour, minute, second, usecond): + w_datetime = PyImport_Import(space, space.wrap("datetime")) + return space.call_method( + w_datetime, "time", + space.wrap(hour), space.wrap(minute), space.wrap(second), + space.wrap(usecond)) Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_datetime.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_datetime.py Mon Apr 19 16:00:57 2010 @@ -0,0 +1,7 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestDatetime(BaseApiTest): + def test_time(self, space, api): + w_time = api.PyTime_FromTime(23, 15, 40, 123456) + assert space.unwrap(space.str(w_time)) == '23:15:40.123456' From afa at codespeak.net Mon Apr 19 16:05:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Apr 2010 16:05:18 +0200 (CEST) Subject: [pypy-svn] r73885 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100419140518.84F5C282B9E@codespeak.net> Author: afa Date: Mon Apr 19 16:05:17 2010 New Revision: 73885 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Log: Remove outdated comment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_import.py Mon Apr 19 16:05:17 2010 @@ -8,7 +8,6 @@ BaseApiTest.setup_method(self, func) def test_import(self, space, api): - # failing because we dont have a caller pdb = api.PyImport_Import(space.wrap("pdb")) assert pdb assert space.getattr(pdb, space.wrap("pm")) From arigo at codespeak.net Mon Apr 19 17:23:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Apr 2010 17:23:50 +0200 (CEST) Subject: [pypy-svn] r73887 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100419152350.39FD0282B9D@codespeak.net> Author: arigo Date: Mon Apr 19 17:23:47 2010 New Revision: 73887 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Log: Start writing the blackhole interpreter. Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Mon Apr 19 17:23:47 2010 @@ -0,0 +1,17 @@ +from pypy.jit.metainterp import history + + +class JitCode(history.AbstractValue): + empty_list = [] + + def __init__(self, name, cfnptr=None, calldescr=None, called_from=None, + graph=None): + self.name = name + self.cfnptr = cfnptr + self.calldescr = calldescr + self.called_from = called_from + self.graph = graph + + def setup(self, code, constants): + self.code = code + self.constants = constants or self.empty_list # share the empty list Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Mon Apr 19 17:23:47 2010 @@ -0,0 +1,102 @@ +from pypy.rlib.unroll import unrolling_iterable +from pypy.tool.sourcetools import func_with_new_name + + +def arguments(*argtypes, **kwds): + resulttype = kwds.pop('returns', None) + assert not kwds + def decorate(function): + function.argtypes = argtypes + function.resulttype = resulttype + return function + return decorate + +class LeaveFrame(Exception): + pass + + +class BlackholeInterpreter(object): + + def __init__(self): + self.registers_i = [0] * 256 + + def setup_insns(self, insns): + assert len(insns) <= 256, "too many instructions!" + self._insns = [None] * len(insns) + for key, value in insns.items(): + assert self._insns[value] is None + self._insns[value] = key + # + all_funcs = [] + for key in self._insns: + assert key is not None, "hole!" + assert key.count('/') == 1, "bad key: %r" % (key,) + name, argcodes = key.split('/') + all_funcs.append(self._get_method(name, argcodes)) + all_funcs = unrolling_iterable(enumerate(all_funcs)) + # + def dispatch(code, position): + opcode = ord(code[position]) + position += 1 + for i, func in all_funcs: + if opcode == i: + return func(code, position) + else: + raise AssertionError("bad opcode") + self.dispatch = dispatch + + def _get_method(self, name, argcodes): + # + def handler(code, position): + args = () + for argcode, argtype in arg_codes_and_types: + if argcode == 'i': + value = self.registers_i[ord(code[position])] + position += 1 + args += (value,) + assert argtype == 'i' + else: + raise AssertionError("bad arg code: %r" % (argcode,)) + result = boundmethod(*args) + if resulttype == 'i': + assert type(result) is int + self.registers_i[ord(code[position])] = result + position += 1 + else: + assert resulttype is None + assert result is None + return position + # + boundmethod = getattr(self, 'opimpl_' + name) + argtypes = boundmethod.argtypes + resulttype = boundmethod.resulttype + if resulttype is not None: + assert argcodes[-1] == 'i' + argcodes = argcodes[:-1] + assert len(argcodes) == len(argtypes) + arg_codes_and_types = unrolling_iterable(zip(argcodes, argtypes)) + handler = func_with_new_name(handler, 'handler_' + name) + return handler + + def setarg_i(self, index, value): + self.registers_i[index] = value + + def run(self, jitcode, position): + code = jitcode.code + constants = jitcode.constants + try: + while True: + position = self.dispatch(code, position) + except LeaveFrame: + pass + + # ---------- + + @arguments("i", "i", returns="i") + def opimpl_int_add(self, a, b): + return a + b + + @arguments("i") + def opimpl_int_return(self, a): + self.result_i = a + raise LeaveFrame Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Mon Apr 19 17:23:47 2010 @@ -0,0 +1,63 @@ +from pypy.jit.codewriter.assembler import Assembler +from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register +from pypy.jit.metainterp.history import ConstInt + + +def test_assemble_simple(): + ssarepr = SSARepr() + i0, i1, i2 = Register(0), Register(1), Register(2) + ssarepr.insns = [ + ('int_add', i0, i1, i2), + ('int_return', i2), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x00\x01\x02" + "\x01\x02") + assert assembler.insns == {'int_add/iii': 0, + 'int_return/i': 1} + +def test_assemble_consts(): + ssarepr = SSARepr() + ssarepr.insns = [ + ('int_return', Register(13)), + ('int_return', Constant(18)), + ('int_return', Constant(-4)), + ('int_return', Constant(128)), + ('int_return', Constant(-129)), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x0D" + "\x01\x12" # use int_return/c for one-byte consts + "\x01\xFC" + "\x00\xFF" # use int_return/i for larger consts + "\x00\xFE") + assert assembler.insns == {'int_return/i': 0, + 'int_return/c': 1} + assert assembler.constants == [ConstInt(-129), ConstInt(128)] + +def test_assemble_loop(): + ssarepr = SSARepr() + i0, i1 = Register(16), Register(17) + ssarepr.insns = [ + (Label('L1'),), + ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4)), + ('int_add', i1, i0, i1), + ('int_sub', i0, Constant(1), i0), + ('goto', TLabel('L1')), + (Label('L2'),), + ('int_return', i1), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x10\x00\x10\x04" + "\x01\x17\x16\x17" + "\x02\x16\x01\x16" + "\x03\x00\x00" + "\x04\x17") + assert assembler.insns == {'goto_if_not_int_gt/Lic': 0, + 'int_add/iii': 1, + 'int_sub/ici': 2, + 'goto/L': 3, + 'int_return/i': 4} Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Mon Apr 19 17:23:47 2010 @@ -0,0 +1,16 @@ +from pypy.jit.codewriter.assembler import JitCode +from pypy.jit.codewriter.blackhole import BlackholeInterpreter + + +def test_simple(): + jitcode = JitCode("test") + jitcode.setup("\x00\x00\x01\x02" + "\x01\x02", + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_add/iii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(0, 40) + blackholeinterp.setarg_i(1, 2) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Mon Apr 19 17:23:47 2010 @@ -8,13 +8,13 @@ ssarepr = SSARepr() i0, i1, i2 = Register(0), Register(1), Register(2) ssarepr.insns = [ - ('livevars', [i0, i1]), + ('foobar', [i0, i1]), ('int_add', i0, i1, i2), ('int_return', i2), ] asm = format_assembler(ssarepr) expected = """ - livevars [%i0, %i1] + foobar [%i0, %i1] int_add %i0, %i1, %i2 int_return %i2 """ @@ -24,7 +24,7 @@ ssarepr = SSARepr() i0, i1 = Register(0), Register(1) ssarepr.insns = [ - ('livevars', [i0, i1]), + ('foobar', [i0, i1]), (Label('L1'),), ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0)), ('int_add', i1, i0, i1), @@ -35,7 +35,7 @@ ] asm = format_assembler(ssarepr) expected = """ - livevars [%i0, %i1] + foobar [%i0, %i1] L1: goto_if_not_int_gt L2, %i0, $0 int_add %i1, %i0, %i1 From xoraxax at codespeak.net Mon Apr 19 18:02:29 2010 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 19 Apr 2010 18:02:29 +0200 (CEST) Subject: [pypy-svn] r73888 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100419160229.617FE282B9D@codespeak.net> Author: xoraxax Date: Mon Apr 19 18:02:28 2010 New Revision: 73888 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Updated TODO file. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Mon Apr 19 18:02:28 2010 @@ -17,6 +17,10 @@ - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). + - Make Numpy work. + + - Export pointers where pointers are expected, i.e. kill the "#define PyExc_OSError &PyExc_OSError" hack. + - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. From arigo at codespeak.net Mon Apr 19 18:03:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Apr 2010 18:03:05 +0200 (CEST) Subject: [pypy-svn] r73889 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100419160305.963BA282B9D@codespeak.net> Author: arigo Date: Mon Apr 19 18:03:04 2010 New Revision: 73889 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Log: Add a few more opcodes. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Mon Apr 19 18:03:04 2010 @@ -1,4 +1,5 @@ from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask, LONG_BIT from pypy.tool.sourcetools import func_with_new_name @@ -14,6 +15,11 @@ class LeaveFrame(Exception): pass +def signedord(c): + value = ord(c) + value = intmask(value << (LONG_BIT-8)) >> (LONG_BIT-8) + return value + class BlackholeInterpreter(object): @@ -35,46 +41,65 @@ all_funcs.append(self._get_method(name, argcodes)) all_funcs = unrolling_iterable(enumerate(all_funcs)) # - def dispatch(code, position): - opcode = ord(code[position]) - position += 1 - for i, func in all_funcs: - if opcode == i: - return func(code, position) - else: - raise AssertionError("bad opcode") - self.dispatch = dispatch + def dispatch_loop(code, position): + while True: + opcode = ord(code[position]) + position += 1 + for i, func in all_funcs: + if opcode == i: + position = func(code, position) + break + else: + raise AssertionError("bad opcode") + self.dispatch_loop = dispatch_loop def _get_method(self, name, argcodes): # def handler(code, position): args = () - for argcode, argtype in arg_codes_and_types: - if argcode == 'i': - value = self.registers_i[ord(code[position])] + next_argcode = 0 + for argtype in argtypes: + if argtype == 'i': + # argcode can be 'i' or 'c'; 'c' stands for a single + # signed byte that gives the value of a small constant. + argcode = argcodes[next_argcode] + next_argcode = next_argcode + 1 + if argcode == 'i': + value = self.registers_i[ord(code[position])] + elif argcode == 'c': + value = signedord(code[position]) + else: + raise AssertionError("bad argcode") position += 1 - args += (value,) - assert argtype == 'i' + elif argtype == 'L': + # argcode should be 'L' too + assert argcodes[next_argcode] == 'L' + next_argcode = next_argcode + 1 + value = ord(code[position]) | (ord(code[position+1])<<8) + position += 2 + elif argtype == 'pc': + value = position else: - raise AssertionError("bad arg code: %r" % (argcode,)) + raise AssertionError("bad argtype") + args += (value,) result = boundmethod(*args) if resulttype == 'i': - assert type(result) is int + # argcode should be 'i' too + assert argcodes[next_argcode] == 'i' + next_argcode = next_argcode + 1 self.registers_i[ord(code[position])] = result position += 1 + elif resulttype == 'L': + position = result else: assert resulttype is None assert result is None + assert next_argcode == len(argcodes) return position # boundmethod = getattr(self, 'opimpl_' + name) - argtypes = boundmethod.argtypes + argtypes = unrolling_iterable(boundmethod.argtypes) resulttype = boundmethod.resulttype - if resulttype is not None: - assert argcodes[-1] == 'i' - argcodes = argcodes[:-1] - assert len(argcodes) == len(argtypes) - arg_codes_and_types = unrolling_iterable(zip(argcodes, argtypes)) handler = func_with_new_name(handler, 'handler_' + name) return handler @@ -85,8 +110,7 @@ code = jitcode.code constants = jitcode.constants try: - while True: - position = self.dispatch(code, position) + self.dispatch_loop(code, position) except LeaveFrame: pass @@ -96,7 +120,22 @@ def opimpl_int_add(self, a, b): return a + b + @arguments("i", "i", returns="i") + def opimpl_int_sub(self, a, b): + return a - b + @arguments("i") def opimpl_int_return(self, a): self.result_i = a raise LeaveFrame + + @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_gt(self, target, a, b, pc): + if a > b: + return pc + else: + return target + + @arguments("L", returns="L") + def opimpl_goto(self, target): + return target Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Mon Apr 19 18:03:04 2010 @@ -14,3 +14,34 @@ blackholeinterp.setarg_i(1, 2) blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 42 + +def test_simple_const(): + jitcode = JitCode("test") + jitcode.setup("\x00\x30\x01\x02" + "\x01\x02", + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_sub/cii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(1, 6) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + +def test_simple_loop(): + jitcode = JitCode("test") + jitcode.setup("\x00\x10\x00\x16\x02" # L1: goto_if_not_int_gt L2, %i0, 2 + "\x01\x17\x16\x17" # int_add %i1, %i0, %i1 + "\x02\x16\x01\x16" # int_sub %i0, $1, %i0 + "\x03\x00\x00" # goto L1 + "\x04\x17", # L2: int_return %i1 + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'goto_if_not_int_gt/Lic': 0, + 'int_add/iii': 1, + 'int_sub/ici': 2, + 'goto/L': 3, + 'int_return/i': 4}) + blackholeinterp.setarg_i(0x16, 6) + blackholeinterp.setarg_i(0x17, 100) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 100+6+5+4+3 From afa at codespeak.net Mon Apr 19 18:21:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Apr 2010 18:21:44 +0200 (CEST) Subject: [pypy-svn] r73890 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100419162144.49392282B9D@codespeak.net> Author: afa Date: Mon Apr 19 18:21:41 2010 New Revision: 73890 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py Log: Fix translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/import_.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/import_.py Mon Apr 19 18:21:41 2010 @@ -37,4 +37,4 @@ @cpython_api([rffi.CCHARP], PyObject) def PyImport_ImportModule(space, name): - return PyImport_Import(space, space.wrap(name)) + return PyImport_Import(space, space.wrap(rffi.charp2str(name))) From benjamin at codespeak.net Mon Apr 19 22:36:46 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 19 Apr 2010 22:36:46 +0200 (CEST) Subject: [pypy-svn] r73891 - pypy/trunk/pypy/module/posix/test Message-ID: <20100419203646.E822C282B9D@codespeak.net> Author: benjamin Date: Mon Apr 19 22:36:44 2010 New Revision: 73891 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: be robust in selecting the encoding on -A Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Mon Apr 19 22:36:44 2010 @@ -647,8 +647,15 @@ def setup_class(cls): cls.space = space cls.w_posix = space.appexec([], GET_POSIX) - cls.save_fs_encoding = space.sys.filesystemencoding - space.sys.filesystemencoding = "utf-8" + if py.test.config.option.runappdirect: + # Can't change encoding + try: + u"?".encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + else: + cls.save_fs_encoding = space.sys.filesystemencoding + space.sys.filesystemencoding = "utf-8" def teardown_class(cls): cls.space.sys.filesystemencoding = cls.save_fs_encoding @@ -659,14 +666,14 @@ self.posix.stat(u"?") except OSError: pass - + def test_open_unicode(self): # Ensure passing unicode doesn't raise UnicodeEncodeError try: self.posix.open(u"?", self.posix.O_WRONLY) except OSError: pass - + def test_remove_unicode(self): # See 2 above ;) try: From benjamin at codespeak.net Mon Apr 19 22:59:50 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 19 Apr 2010 22:59:50 +0200 (CEST) Subject: [pypy-svn] r73892 - pypy/build/bot2/pypybuildbot Message-ID: <20100419205950.BF438282B9D@codespeak.net> Author: benjamin Date: Mon Apr 19 22:59:49 2010 New Revision: 73892 Modified: pypy/build/bot2/pypybuildbot/master.py Log: run 64 bit app level tests Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Apr 19 22:59:49 2010 @@ -112,6 +112,7 @@ LINUX64 = "own-linux-x86-64" MACOSX32 = "own-macosx-x86-32" APPLVLLINUX32 = "pypy-c-app-level-linux-x86-32" +APPLVLLINUX64 = "pypy-c-app-level-linux-64" STACKLESSAPPLVLLINUX32 = "pypy-c-stackless-app-level-linux-x86-32" APPLVLWIN32 = "pypy-c-app-level-win-32" @@ -132,7 +133,7 @@ 'schedulers': [ Nightly("nightly-first", [LINUX32, LINUX64], hour=4, minute=44), - Nightly("nightly", [APPLVLLINUX32, APPLVLWIN32, + Nightly("nightly", [APPLVLLINUX32, APPLVLLINUX64, APPLVLWIN32, STACKLESSAPPLVLLINUX32, JITLINUX32, OJITLINUX32, MACOSX32], hour=4, minute=45), @@ -170,6 +171,12 @@ "factory": pypyTranslatedAppLevelTestFactory, 'category': 'applevel' }, + {"name": APPLVLLINUX64, + "slavenames": ["tannit64"], + "builddir": APPLVLLINUX64, + "factory": pypyTranslatedAppLevelTestFactory, + "category": "applevel" + }, {"name": STACKLESSAPPLVLLINUX32, "slavenames": ["bigdogvm1", "tannit32"], "builddir": STACKLESSAPPLVLLINUX32, From afa at codespeak.net Tue Apr 20 00:27:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 00:27:01 +0200 (CEST) Subject: [pypy-svn] r73893 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100419222701.DF6AF282B9D@codespeak.net> Author: afa Date: Tue Apr 20 00:27:00 2010 New Revision: 73893 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h Log: Support Py_UNICODE_WIDE, this fixes test_sre Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h Tue Apr 20 00:27:00 2010 @@ -20,6 +20,7 @@ #define HAVE_USABLE_WCHAR_T 1 #ifndef _WIN32 #define Py_UNICODE_SIZE 4 +#define Py_UNICODE_WIDE #else #define Py_UNICODE_SIZE 2 #endif From afa at codespeak.net Tue Apr 20 08:55:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 08:55:00 +0200 (CEST) Subject: [pypy-svn] r73894 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420065500.49F7E282BAD@codespeak.net> Author: afa Date: Tue Apr 20 08:54:58 2010 New Revision: 73894 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: add PyObject_IsInstance and PyObject_IsSubclass Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Tue Apr 20 08:54:58 2010 @@ -196,3 +196,29 @@ from pypy.objspace.descroperation import object_getattribute w_descr = object_getattribute(space) return space.get_and_call_function(w_descr, w_obj, w_name) + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_IsInstance(space, w_inst, w_cls): + """Returns 1 if inst is an instance of the class cls or a subclass of + cls, or 0 if not. On error, returns -1 and sets an exception. If + cls is a type object rather than a class object, PyObject_IsInstance() + returns 1 if inst is of type cls. If cls is a tuple, the check will + be done against every entry in cls. The result will be 1 when at least one + of the checks returns 1, otherwise it will be 0. If inst is not a class + instance and cls is neither a type object, nor a class object, nor a + tuple, inst must have a __class__ attribute --- the class relationship + of the value of that attribute with cls will be used to determine the result + of this function.""" + from pypy.module.__builtin__.abstractinst import abstract_isinstance_w + return abstract_isinstance_w(space, w_inst, w_cls) + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_IsSubclass(space, w_derived, w_cls): + """Returns 1 if the class derived is identical to or derived from the class + cls, otherwise returns 0. In case of an error, returns -1. If cls + is a tuple, the check will be done against every entry in cls. The result will + be 1 when at least one of the checks returns 1, otherwise it will be + 0. If either derived or cls is not an actual class object (or tuple), + this function uses the generic algorithm described above.""" + from pypy.module.__builtin__.abstractinst import abstract_issubclass_w + return abstract_issubclass_w(space, w_derived, w_cls) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 08:54:58 2010 @@ -4429,38 +4429,6 @@ function.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) -def PyObject_IsInstance(space, inst, cls): - """Returns 1 if inst is an instance of the class cls or a subclass of - cls, or 0 if not. On error, returns -1 and sets an exception. If - cls is a type object rather than a class object, PyObject_IsInstance() - returns 1 if inst is of type cls. If cls is a tuple, the check will - be done against every entry in cls. The result will be 1 when at least one - of the checks returns 1, otherwise it will be 0. If inst is not a - class instance and cls is neither a type object, nor a class object, nor a - tuple, inst must have a __class__ attribute --- the class relationship - of the value of that attribute with cls will be used to determine the result - of this function. - - - - Support for a tuple as the second argument added.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real) -def PyObject_IsSubclass(space, derived, cls): - """Returns 1 if the class derived is identical to or derived from the class - cls, otherwise returns 0. In case of an error, returns -1. If cls - is a tuple, the check will be done against every entry in cls. The result will - be 1 when at least one of the checks returns 1, otherwise it will be - 0. If either derived or cls is not an actual class object (or tuple), - this function uses the generic algorithm described above. - - - - Older versions of Python did not support a tuple as the second argument.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real) def PyCallable_Check(space, o): """Determine if the object o is callable. Return 1 if the object is callable Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Tue Apr 20 08:54:58 2010 @@ -105,3 +105,22 @@ assert api.PyObject_TypeCheck(space.wrap(True), api.PyBool_Type) assert api.PyObject_TypeCheck(space.wrap(1.2), api.PyFloat_Type) assert api.PyObject_TypeCheck(space.w_int, api.PyType_Type) + + def test_IsInstance(self, space, api): + assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1 + assert api.PyObject_IsInstance(space.wrap(1), space.w_float) == 0 + assert api.PyObject_IsInstance(space.w_True, space.w_int) == 1 + assert api.PyObject_IsInstance( + space.wrap(1), space.newtuple([space.w_int, space.w_float])) == 1 + assert api.PyObject_IsInstance(space.w_type, space.w_type) == 1 + assert api.PyObject_IsInstance(space.wrap(1), space.w_None) == -1 + api.PyErr_Clear() + + def test_IsSubclass(self, space, api): + assert api.PyObject_IsSubclass(space.w_type, space.w_type) == 1 + assert api.PyObject_IsSubclass(space.w_type, space.w_object) == 1 + assert api.PyObject_IsSubclass(space.w_object, space.w_type) == 0 + assert api.PyObject_IsSubclass( + space.w_type, space.newtuple([space.w_int, space.w_type])) == 1 + assert api.PyObject_IsSubclass(space.wrap(1), space.w_type) == -1 + api.PyErr_Clear() From agaynor at codespeak.net Tue Apr 20 09:03:12 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 20 Apr 2010 09:03:12 +0200 (CEST) Subject: [pypy-svn] r73895 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420070312.48371282BD4@codespeak.net> Author: agaynor Date: Tue Apr 20 09:03:10 2010 New Revision: 73895 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: Implement PyList_Sort Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Tue Apr 20 09:03:10 2010 @@ -83,3 +83,10 @@ raise OperationError(space.w_TypeError, space.wrap("expected list object")) return PyList_GET_SIZE(space, ref) + + at cpython_api([PyObject], rffi.INT_real, error=-1) +def PyList_Sort(space, w_list): + """Sort the items of list in place. Return 0 on success, -1 on + failure. This is equivalent to list.sort().""" + space.call_method(w_list, "sort") + return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 09:03:10 2010 @@ -3385,12 +3385,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real) -def PyList_Sort(space, list): - """Sort the items of list in place. Return 0 on success, -1 on - failure. This is equivalent to list.sort().""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) def PyList_Reverse(space, list): """Reverse the items of list in place. Return 0 on success, -1 on failure. This is the equivalent of list.reverse().""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Tue Apr 20 09:03:10 2010 @@ -32,6 +32,11 @@ assert api.PyList_Size(space.w_None) == -1 assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + + def test_sort(self, space, api): + l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)]) + assert api.PyList_Sort(l) == 0 + assert space.eq_w(l, space.newlist([space.wrap(0), space.wrap(1), space.wrap(7000)])) class AppTestListObject(AppTestCpythonExtensionBase): def test_listobject(self): From agaynor at codespeak.net Tue Apr 20 09:06:06 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 20 Apr 2010 09:06:06 +0200 (CEST) Subject: [pypy-svn] r73896 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420070606.5263E282BD4@codespeak.net> Author: agaynor Date: Tue Apr 20 09:06:04 2010 New Revision: 73896 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: Implement PyList_Reverse Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Tue Apr 20 09:06:04 2010 @@ -90,3 +90,11 @@ failure. This is equivalent to list.sort().""" space.call_method(w_list, "sort") return 0 + + at cpython_api([PyObject], rffi.INT_real, error=-1) +def PyList_Reverse(space, w_list): + """Reverse the items of list in place. Return 0 on success, -1 on + failure. This is the equivalent of list.reverse().""" + space.call_method(w_list, "reverse") + return 0 + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 09:06:04 2010 @@ -3384,12 +3384,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyList_Reverse(space, list): - """Reverse the items of list in place. Return 0 on success, -1 on - failure. This is the equivalent of list.reverse().""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyList_AsTuple(space, list): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Tue Apr 20 09:06:04 2010 @@ -37,6 +37,11 @@ l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)]) assert api.PyList_Sort(l) == 0 assert space.eq_w(l, space.newlist([space.wrap(0), space.wrap(1), space.wrap(7000)])) + + def test_reverse(self, space, api): + l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)]) + assert api.PyList_Reverse(l) == 0 + assert space.eq_w(l, space.newlist([space.wrap(1), space.wrap(2), space.wrap(3)])) class AppTestListObject(AppTestCpythonExtensionBase): def test_listobject(self): From agaynor at codespeak.net Tue Apr 20 09:07:42 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 20 Apr 2010 09:07:42 +0200 (CEST) Subject: [pypy-svn] r73897 - in pypy/branch/cpython-extension: lib-python/modified-2.5.2/distutils pypy/module/cpyext Message-ID: <20100420070742.B487F282BD4@codespeak.net> Author: agaynor Date: Tue Apr 20 09:07:41 2010 New Revision: 73897 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Log: Implement some error checking I forgot. Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Tue Apr 20 09:07:41 2010 @@ -7,15 +7,18 @@ from distutils.errors import DistutilsPlatformError -PYPY_PREFIX = os.path.normpath(sys.pypy_prefix) +if hasattr(sys, "pypy_prefix"): + PYPY_PREFIX = os.path.normpath(sys.pypy_prefix) +else: + PYPY_PREFIX = "/home/alex/projects/pypy-cpy/" python_build = False def get_python_inc(plat_specific=0, prefix=None): from os.path import join as j if plat_specific: - return j(sys.pypy_prefix, "pypy", "_interfaces") - return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') + return j(PYPY_PREFIX, "pypy", "_interfaces") + return j(PYPY_PREFIX, 'pypy', 'module', 'cpyext', 'include') def get_python_version(): """Return a string containing the major and minor Python version, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Tue Apr 20 09:07:41 2010 @@ -88,6 +88,8 @@ def PyList_Sort(space, w_list): """Sort the items of list in place. Return 0 on success, -1 on failure. This is equivalent to list.sort().""" + if not isinstance(w_list, W_ListObject): + PyErr_BadInternalCall(space) space.call_method(w_list, "sort") return 0 @@ -95,6 +97,8 @@ def PyList_Reverse(space, w_list): """Reverse the items of list in place. Return 0 on success, -1 on failure. This is the equivalent of list.reverse().""" + if not isinstance(w_list, W_ListObject): + PyErr_BadInternalCall(space) space.call_method(w_list, "reverse") return 0 From agaynor at codespeak.net Tue Apr 20 09:09:17 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 20 Apr 2010 09:09:17 +0200 (CEST) Subject: [pypy-svn] r73898 - pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils Message-ID: <20100420070917.99530282BD4@codespeak.net> Author: agaynor Date: Tue Apr 20 09:09:16 2010 New Revision: 73898 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Log: Revert change I accidentally made, stupid local changes... Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Tue Apr 20 09:09:16 2010 @@ -7,18 +7,15 @@ from distutils.errors import DistutilsPlatformError -if hasattr(sys, "pypy_prefix"): - PYPY_PREFIX = os.path.normpath(sys.pypy_prefix) -else: - PYPY_PREFIX = "/home/alex/projects/pypy-cpy/" +PYPY_PREFIX = os.path.normpath(sys.pypy_prefix) python_build = False def get_python_inc(plat_specific=0, prefix=None): from os.path import join as j if plat_specific: - return j(PYPY_PREFIX, "pypy", "_interfaces") - return j(PYPY_PREFIX, 'pypy', 'module', 'cpyext', 'include') + return j(sys.pypy_prefix, "pypy", "_interfaces") + return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') def get_python_version(): """Return a string containing the major and minor Python version, From afa at codespeak.net Tue Apr 20 11:07:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 11:07:24 +0200 (CEST) Subject: [pypy-svn] r73899 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420090724.51712282BD4@codespeak.net> Author: afa Date: Tue Apr 20 11:07:22 2010 New Revision: 73899 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: Add PyLong <-> LongLong conversions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Tue Apr 20 11:07:22 2010 @@ -11,7 +11,19 @@ """Return a new PyLongObject object from v, or NULL on failure.""" return space.newlong(val) - at cpython_api([PyObject], rffi.ULONG, error=0) + at cpython_api([rffi.LONGLONG], PyObject) +def PyLong_FromLongLong(space, val): + """Return a new PyLongObject object from a C long long, or NULL + on failure.""" + return space.wrap(val) + + at cpython_api([rffi.ULONGLONG], PyObject) +def PyLong_FromUnsignedLongLong(space, val): + """Return a new PyLongObject object from a C unsigned long long, + or NULL on failure.""" + return space.wrap(val) + + at cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLong(space, w_long): """ Return a C unsigned long representation of the contents of pylong. @@ -19,6 +31,22 @@ raised.""" return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + at cpython_api([PyObject], rffi.LONGLONG, error=-1) +def PyLong_AsLongLong(space, w_long): + """ + Return a C unsigned long representation of the contents of pylong. + If pylong is greater than ULONG_MAX, an OverflowError is + raised.""" + return rffi.cast(rffi.LONGLONG, space.r_longlong_w(w_long)) + + at cpython_api([PyObject], rffi.ULONGLONG, error=-1) +def PyLong_AsUnsignedLongLong(space, w_long): + """ + Return a C unsigned long representation of the contents of pylong. + If pylong is greater than ULONG_MAX, an OverflowError is + raised.""" + return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): """Create a Python integer or long integer from the pointer p. The pointer value Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 11:07:22 2010 @@ -3429,18 +3429,6 @@ """ raise NotImplementedError - at cpython_api([{PY_LONG_LONG}], PyObject) -def PyLong_FromLongLong(space, v): - """Return a new PyLongObject object from a C long long, or NULL - on failure.""" - raise NotImplementedError - - at cpython_api([{unsigned PY_LONG_LONG}], PyObject) -def PyLong_FromUnsignedLongLong(space, v): - """Return a new PyLongObject object from a C unsigned long long, - or NULL on failure.""" - raise NotImplementedError - @cpython_api([{double}], PyObject) def PyLong_FromDouble(space, v): """Return a new PyLongObject object from the integer part of v, or Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Tue Apr 20 11:07:22 2010 @@ -4,6 +4,7 @@ from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.longobject import W_LongObject from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestLongObject(BaseApiTest): @@ -39,3 +40,27 @@ l = space.call_function(L) assert api.PyLong_Check(l) assert not api.PyLong_CheckExact(l) + + def test_as_longlong(self, space, api): + assert api.PyLong_AsLongLong(space.wrap(1<<62)) == 1<<62 + assert api.PyLong_AsLongLong(space.wrap(1<<63)) == -1 + api.PyErr_Clear() + + assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<63)) == 1<<63 + assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<64)) == (1<<64) - 1 + assert api.PyErr_Occurred() + api.PyErr_Clear() + +class AppTestLongObject(AppTestCpythonExtensionBase): + def test_fromlonglong(self): + module = self.import_extension('foo', [ + ("from_longlong", "METH_NOARGS", + """ + return PyLong_FromLongLong((long long)-1); + """), + ("from_unsignedlonglong", "METH_NOARGS", + """ + return PyLong_FromUnsignedLongLong((unsigned long long)-1); + """)]) + assert module.from_longlong() == -1 + assert module.from_unsignedlonglong() == (1<<64) - 1 From afa at codespeak.net Tue Apr 20 12:00:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 12:00:57 +0200 (CEST) Subject: [pypy-svn] r73900 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src test Message-ID: <20100420100057.9FE37282BD4@codespeak.net> Author: afa Date: Tue Apr 20 12:00:55 2010 New Revision: 73900 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/eval.py pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Log: Add PyObject_CallFunction and PyObject_CallMethod Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 20 12:00:55 2010 @@ -241,6 +241,8 @@ 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', + 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', + 'PyObject_CallMethod', 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', '_PyArg_NoKeywords', 'PyObject_AsReadBuffer', 'PyObject_CheckReadBuffer', ] Modified: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/eval.py Tue Apr 20 12:00:55 2010 @@ -14,3 +14,14 @@ of the Python expression apply(callable_object, args) or callable_object(*args).""" return space.call(w_obj, w_arg) + + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyObject_Call(space, w_obj, w_args, w_kw): + """ + Call a callable Python object, with arguments given by the + tuple args, and named arguments given by the dictionary kw. If no named + arguments are needed, kw may be NULL. args must not be NULL, use an + empty tuple if no arguments are needed. Returns the result of the call on + success, or NULL on failure. This is the equivalent of the Python expression + apply(callable_object, args, kw) or callable_object(*args, **kw).""" + return space.call(w_obj, w_args, w_kw) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h Tue Apr 20 12:00:55 2010 @@ -12,6 +12,11 @@ #define PyEval_CallObject(func,arg) \ PyEval_CallObjectWithKeywords(func, arg, (PyObject *)NULL) +PyObject * PyEval_CallFunction(PyObject *obj, const char *format, ...); +PyObject * PyEval_CallMethod(PyObject *obj, const char *name, const char *format, ...); +PyObject * PyObject_CallFunction(PyObject *obj, char *format, ...); +PyObject * PyObject_CallMethod(PyObject *obj, char *name, char *format, ...); + #ifdef __cplusplus } #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c Tue Apr 20 12:00:55 2010 @@ -518,6 +518,80 @@ return res; } +static PyObject* +call_function_tail(PyObject *callable, PyObject *args) +{ + PyObject *retval; + + if (args == NULL) + return NULL; + + if (!PyTuple_Check(args)) { + PyObject *a; + + a = PyTuple_New(1); + if (a == NULL) { + Py_DECREF(args); + return NULL; + } + PyTuple_SET_ITEM(a, 0, args); + args = a; + } + retval = PyObject_Call(callable, args, NULL); + + Py_DECREF(args); + + return retval; +} + +PyObject * +PyObject_CallFunction(PyObject *callable, char *format, ...) +{ + va_list va; + PyObject *args; + + if (format && *format) { + va_start(va, format); + args = Py_VaBuildValue(format, va); + va_end(va); + } + else + args = PyTuple_New(0); + + return call_function_tail(callable, args); +} + +PyObject * +PyObject_CallMethod(PyObject *o, char *name, char *format, ...) +{ + va_list va; + PyObject *args; + PyObject *func = NULL; + PyObject *retval = NULL; + + func = PyObject_GetAttrString(o, name); + if (func == NULL) { + PyErr_SetString(PyExc_AttributeError, name); + return 0; + } + + if (format && *format) { + va_start(va, format); + args = Py_VaBuildValue(format, va); + va_end(va); + } + else + args = PyTuple_New(0); + + retval = call_function_tail(func, args); + + exit: + /* args gets consumed in call_function_tail */ + Py_XDECREF(func); + + return retval; +} + int PyModule_AddObject(PyObject *m, const char *name, PyObject *o) { Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 12:00:55 2010 @@ -4411,21 +4411,6 @@ and 0 otherwise. This function always succeeds.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyObject_Call(space, callable_object, args, kw): - """ - - - - Call a callable Python object callable_object, with arguments given by the - tuple args, and named arguments given by the dictionary kw. If no named - arguments are needed, kw may be NULL. args must not be NULL, use an - empty tuple if no arguments are needed. Returns the result of the call on - success, or NULL on failure. This is the equivalent of the Python expression - apply(callable_object, args, kw) or callable_object(*args, **kw). - """ - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, ...], PyObject) def PyObject_CallFunction(space, callable, format, ): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Tue Apr 20 12:00:55 2010 @@ -1,4 +1,4 @@ - +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest class TestEval(BaseApiTest): @@ -55,3 +55,22 @@ w_res = api.PyObject_CallObject(w_f, w_t) assert space.int_w(w_res) == 10 + +class AppTestCall(AppTestCpythonExtensionBase): + def test_CallFunction(self): + module = self.import_extension('foo', [ + ("call_func", "METH_VARARGS", + """ + return PyObject_CallFunction(PyTuple_GetItem(args, 0), + "siO", "text", 42, Py_None); + """), + ("call_method", "METH_VARARGS", + """ + return PyObject_CallMethod(PyTuple_GetItem(args, 0), + "count", "s", "t"); + """), + ]) + def f(*args): + return args + assert module.call_func(f) == ("text", 42, None) + assert module.call_method("text") == 2 From arigo at codespeak.net Tue Apr 20 12:03:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 12:03:34 +0200 (CEST) Subject: [pypy-svn] r73901 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100420100334.E89A2282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 12:03:33 2010 New Revision: 73901 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Log: Finish the basic of the "assembler" pass. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Tue Apr 20 12:03:33 2010 @@ -1,4 +1,7 @@ from pypy.jit.metainterp import history +from pypy.jit.codewriter.flatten import Register, Label, TLabel +from pypy.objspace.flow.model import Constant +from pypy.jit.metainterp.history import ConstInt class JitCode(history.AbstractValue): @@ -15,3 +18,61 @@ def setup(self, code, constants): self.code = code self.constants = constants or self.empty_list # share the empty list + + +class Assembler(object): + + def __init__(self): + self.insns = {} + + def assemble(self, ssarepr): + code = [] + constants_dict = {} + constants_rev = [] + label_positions = {} + tlabel_positions = [] + for insn in ssarepr.insns: + if isinstance(insn[0], Label): + label_positions[insn[0].name] = len(code) + continue + startposition = len(code) + code.append("temporary placeholder") + # + argcodes = [] + for x in insn[1:]: + if isinstance(x, Register): + code.append(chr(x.index)) + argcodes.append('i') + elif isinstance(x, Constant): + if -128 <= x.value <= 127: + code.append(chr(x.value & 0xFF)) + argcodes.append('c') + else: + if x not in constants_dict: + constants_rev.append(ConstInt(x.value)) + constants_dict[x] = 256 - len(constants_rev) + code.append(chr(constants_dict[x])) + argcodes.append('i') + elif isinstance(x, TLabel): + tlabel_positions.append((x.name, len(code))) + code.append("temp 1") + code.append("temp 2") + argcodes.append('L') + else: + raise NotImplementedError(x) + # + key = insn[0] + '/' + ''.join(argcodes) + num = self.insns.setdefault(key, len(self.insns)) + code[startposition] = chr(num) + # + for name, pos in tlabel_positions: + assert code[pos ] == "temp 1" + assert code[pos+1] == "temp 2" + target = label_positions[name] + assert 0 <= target <= 0xFFFF + code[pos ] = chr(target & 0xFF) + code[pos+1] = chr(target >> 8) + # + jitcode = JitCode(ssarepr.name) + jitcode.setup(''.join(code), constants_rev[::-1]) + return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 20 12:03:33 2010 @@ -2,7 +2,8 @@ class SSARepr(object): - def __init__(self): + def __init__(self, name): + self.name = name self.insns = [] class Label(object): @@ -54,7 +55,7 @@ assert self.regalloc.getcolor(inputargs[i]) == i def generate_ssa_form(self): - self.assembler = SSARepr() + self.assembler = SSARepr(self.graph.name) self.seen_blocks = {} self.make_bytecode_block(self.graph.startblock) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Tue Apr 20 12:03:33 2010 @@ -1,10 +1,11 @@ from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register +from pypy.objspace.flow.model import Constant from pypy.jit.metainterp.history import ConstInt def test_assemble_simple(): - ssarepr = SSARepr() + ssarepr = SSARepr("test") i0, i1, i2 = Register(0), Register(1), Register(2) ssarepr.insns = [ ('int_add', i0, i1, i2), @@ -18,7 +19,7 @@ 'int_return/i': 1} def test_assemble_consts(): - ssarepr = SSARepr() + ssarepr = SSARepr("test") ssarepr.insns = [ ('int_return', Register(13)), ('int_return', Constant(18)), @@ -35,11 +36,11 @@ "\x00\xFE") assert assembler.insns == {'int_return/i': 0, 'int_return/c': 1} - assert assembler.constants == [ConstInt(-129), ConstInt(128)] + assert jitcode.constants == [ConstInt(-129), ConstInt(128)] def test_assemble_loop(): - ssarepr = SSARepr() - i0, i1 = Register(16), Register(17) + ssarepr = SSARepr("test") + i0, i1 = Register(0x16), Register(0x17) ssarepr.insns = [ (Label('L1'),), ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4)), @@ -51,7 +52,7 @@ ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode.code == ("\x00\x10\x00\x10\x04" + assert jitcode.code == ("\x00\x10\x00\x16\x04" "\x01\x17\x16\x17" "\x02\x16\x01\x16" "\x03\x00\x00" Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Tue Apr 20 12:03:33 2010 @@ -5,7 +5,7 @@ def test_format_assembler_simple(): - ssarepr = SSARepr() + ssarepr = SSARepr("test") i0, i1, i2 = Register(0), Register(1), Register(2) ssarepr.insns = [ ('foobar', [i0, i1]), @@ -21,7 +21,7 @@ assert asm == str(py.code.Source(expected)).strip() + '\n' def test_format_assembler_loop(): - ssarepr = SSARepr() + ssarepr = SSARepr("test") i0, i1 = Register(0), Register(1) ssarepr.insns = [ ('foobar', [i0, i1]), From arigo at codespeak.net Tue Apr 20 13:44:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 13:44:01 +0200 (CEST) Subject: [pypy-svn] r73902 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100420114401.5A864282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 13:43:59 2010 New Revision: 73902 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (contents, props changed) Log: Integration code and test. Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Tue Apr 20 13:43:59 2010 @@ -0,0 +1,22 @@ +from pypy.jit.codewriter import support +from pypy.jit.codewriter.regalloc import perform_register_allocation +from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.assembler import Assembler + + +class CodeWriter(object): + + def __init__(self): + self.assembler = Assembler() + + def transform_func_to_jitcode(self, func, values, type_system='lltype'): + """For testing.""" + rtyper = support.annotate(func, values, type_system=type_system) + graph = rtyper.annotator.translator.graphs[0] + return self.transform_graph_to_jitcode(graph) + + def transform_graph_to_jitcode(self, graph): + regalloc = perform_register_allocation(graph) + ssarepr = flatten_graph(graph, regalloc) + jitcode = self.assembler.assemble(ssarepr) + return jitcode Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Tue Apr 20 13:43:59 2010 @@ -0,0 +1,23 @@ +from pypy.jit.codewriter.codewriter import CodeWriter + + +def test_loop(): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + cw = CodeWriter() + jitcode = cw.transform_func_to_jitcode(f, [5, 6]) + assert jitcode.code == ("\x00\x00\x00\x02" + "\x01\x13\x00\x02" + "\x02\x01\x00\x01" + "\x03\x00\x01\x00" + "\x04\x00\x00" + "\x05\x01") + assert cw.assembler.insns == {'int_gt/ici': 0, + 'goto_if_not/Li': 1, + 'int_add/iii': 2, + 'int_sub/ici': 3, + 'goto/L': 4, + 'int_return/i': 5} From afa at codespeak.net Tue Apr 20 13:57:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 13:57:25 +0200 (CEST) Subject: [pypy-svn] r73903 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420115725.B1DD8282BD4@codespeak.net> Author: afa Date: Tue Apr 20 13:57:23 2010 New Revision: 73903 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/import_.py pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Introduce and use CONST_STRING, identical to rffi.CCHARP except that it will be rendered as "const char*" in pypy_decl.h Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 20 13:57:23 2010 @@ -56,6 +56,8 @@ _compilation_info_ = CConfig._compilation_info_ VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list') +CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) +assert CONST_STRING is not rffi.CCHARP constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING @@ -566,7 +568,10 @@ restype = db.gettype(func.restype).replace('@', '').strip() args = [] for i, argtype in enumerate(func.argtypes): - arg = db.gettype(argtype) + if argtype is CONST_STRING: + arg = 'const char *@' + else: + arg = db.gettype(argtype) arg = arg.replace('@', 'arg%d' % (i,)).strip() args.append(arg) args = ', '.join(args) or "void" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Tue Apr 20 13:57:23 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, build_type_checkers,\ - Py_ssize_t, PyObjectP + Py_ssize_t, PyObjectP, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError @@ -26,7 +26,7 @@ else: PyErr_BadInternalCall(space) - at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real, error=-1) + at cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyDict_SetItemString(space, w_dict, key_ptr, w_obj): if PyDict_Check(space, w_dict): key = rffi.charp2str(key_ptr) @@ -37,7 +37,7 @@ else: PyErr_BadInternalCall(space) - at cpython_api([PyObject, rffi.CCHARP], PyObject, borrowed=True) + at cpython_api([PyObject, CONST_STRING], PyObject, borrowed=True) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/import_.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/import_.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/import_.py Tue Apr 20 13:57:23 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import module -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject +from pypy.module.cpyext.api import ( + generic_cpy_call, cpython_api, PyObject, CONST_STRING) from pypy.rpython.lltypesystem import rffi from pypy.interpreter.error import OperationError @@ -35,6 +36,6 @@ [w_name, w_globals, w_globals, space.newlist([space.wrap("__doc__")])])) - at cpython_api([rffi.CCHARP], PyObject) + at cpython_api([CONST_STRING], PyObject) def PyImport_ImportModule(space, name): return PyImport_Import(space, space.wrap(rffi.charp2str(name))) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Tue Apr 20 13:57:23 2010 @@ -9,7 +9,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ - cpython_struct, METH_KEYWORDS, METH_O + cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated @@ -189,7 +189,7 @@ return space.wrap(W_PyCWrapperObject(space, pto, method_name, wrapper_func, doc, flags, func)) - at cpython_api([lltype.Ptr(PyMethodDef), PyObject, rffi.CCHARP], PyObject) + at cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) def Py_FindMethod(space, table, w_ob, name_ptr): """Return a bound method object for an extension type implemented in C. This can be useful in the implementation of a tp_getattro or Modified: pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/modsupport.py Tue Apr 20 13:57:23 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction @@ -15,7 +15,7 @@ space.setitem(w_modules, w_name, w_mod) return w_mod - at cpython_api([rffi.CCHARP, lltype.Ptr(PyMethodDef), rffi.CCHARP, + at cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, PyObject, rffi.INT_real], PyObject, borrowed=False) # we cannot borrow here def Py_InitModule4(space, name, methods, doc, w_self, apiver): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Tue Apr 20 13:57:23 2010 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ - Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE + Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State @@ -70,7 +70,7 @@ def PyObject_Not(space, w_obj): return not space.is_true(w_obj) - at cpython_api([PyObject, rffi.CCHARP], PyObject) + at cpython_api([PyObject, CONST_STRING], PyObject) def PyObject_GetAttrString(space, w_obj, name_ptr): """Retrieve an attribute named attr_name from object o. Returns the attribute value on success, or NULL on failure. This is the equivalent of the Python Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 20 13:57:23 2010 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import PyObject, make_ref, register_container from pypy.module.cpyext.state import State @@ -15,7 +15,7 @@ state = space.fromcache(State) state.set_exception(w_type, w_value) - at cpython_api([PyObject, rffi.CCHARP], lltype.Void) + at cpython_api([PyObject, CONST_STRING], lltype.Void) def PyErr_SetString(space, w_type, message_ptr): message = rffi.charp2str(message_ptr) PyErr_SetObject(space, w_type, space.wrap(message)) @@ -114,7 +114,7 @@ return PyErr_GivenExceptionMatches(space, w_type, w_exc) - at cpython_api([PyObject, rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) + at cpython_api([PyObject, CONST_STRING, rffi.INT_real], rffi.INT_real, error=-1) def PyErr_WarnEx(space, w_category, message_ptr, stacklevel): """Issue a warning message. The category argument is a warning category (see below) or NULL; the message argument is a message string. stacklevel is a @@ -155,7 +155,7 @@ os.write(2, "WARNING: " + message + "\n") return 0 - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) + at cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) def PyErr_Warn(space, w_category, message): """Issue a warning message. The category argument is a warning category (see below) or NULL; the message argument is a message string. The warning will Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Tue Apr 20 13:57:23 2010 @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t +from pypy.module.cpyext.api import ( + cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) from pypy.module.cpyext.pyobject import PyObject, register_container from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject @@ -21,7 +22,7 @@ return space.int_w(space.len(w_obj)) - at cpython_api([PyObject, rffi.CCHARP], PyObject) + at cpython_api([PyObject, CONST_STRING], PyObject) def PySequence_Fast(space, w_obj, m): """Returns the sequence o as a tuple, unless it is already a tuple or list, in which case o is returned. Use PySequence_Fast_GET_ITEM() to access the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Tue Apr 20 13:57:23 2010 @@ -1,9 +1,9 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, bootstrap_function, PyVarObjectFields, - Py_ssize_t, cpython_struct, PyObjectFields, - ADDR, CANNOT_FAIL, build_type_checkers, - PyObjectP, PyTypeObjectPtr, generic_cpy_call) +from pypy.module.cpyext.api import ( + cpython_api, bootstrap_function, PyVarObjectFields, Py_ssize_t, + cpython_struct, PyObjectFields, ADDR, CONST_STRING, CANNOT_FAIL, + build_type_checkers, PyObjectP, PyTypeObjectPtr, generic_cpy_call) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr from pypy.module.cpyext.state import State @@ -61,7 +61,7 @@ pto = rffi.cast(PyObject, pto) Py_DecRef(space, pto) - at cpython_api([rffi.CCHARP, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) + at cpython_api([CONST_STRING, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) def PyString_FromStringAndSize(space, char_p, length): if char_p: s = rffi.charpsize2str(char_p, length) @@ -70,7 +70,7 @@ else: return new_empty_str(space, length) - at cpython_api([rffi.CCHARP], PyObject) + at cpython_api([CONST_STRING], PyObject) def PyString_FromString(space, char_p): s = rffi.charp2str(char_p) return space.wrap(s) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 20 13:57:23 2010 @@ -1,10 +1,10 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb -from pypy.module.cpyext.api import (CANNOT_FAIL, Py_ssize_t, - build_type_checkers, cpython_api, - bootstrap_function, generic_cpy_call, - PyObjectFields, cpython_struct) +from pypy.module.cpyext.api import ( + CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, + bootstrap_function, generic_cpy_call, PyObjectFields, + cpython_struct, CONST_STRING) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr from pypy.module.cpyext.stringobject import PyString_Check @@ -153,7 +153,7 @@ i += 1 return default_encoding - at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1) + at cpython_api([CONST_STRING], rffi.INT_real, error=-1) def PyUnicode_SetDefaultEncoding(space, encoding): """Sets the currently active default encoding. Returns 0 on success, -1 in case of an error.""" @@ -162,7 +162,7 @@ default_encoding[0] = '\x00' return 0 - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) + at cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) def PyUnicode_AsEncodedString(space, w_unicode, encoding, errors): """Encode a Unicode object and return the result as Python string object. encoding and errors have the same meaning as the parameters of the same name @@ -193,6 +193,6 @@ ptr = make_ref(space, space.wrap(s)) return ptr - at cpython_api([PyObject, rffi.CCHARP], PyObject) + at cpython_api([PyObject, CONST_STRING], PyObject) def _PyUnicode_AsDefaultEncodedString(space, w_unicode, errors): return PyUnicode_AsEncodedString(space, w_unicode, lltype.nullptr(rffi.CCHARP.TO), errors) From afa at codespeak.net Tue Apr 20 14:16:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 14:16:44 +0200 (CEST) Subject: [pypy-svn] r73904 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420121644.73F1B282BD4@codespeak.net> Author: afa Date: Tue Apr 20 14:16:42 2010 New Revision: 73904 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: PyString_FromStringAndSize must return a PyObject*, not PyStringObject* Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Tue Apr 20 14:16:42 2010 @@ -28,8 +28,8 @@ py_str.c_ob_refcnt = 1 buflen = length + 1 - py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') - py_str.c_buffer[buflen-1] = '\0' + py_str.c_buffer = lltype.malloc(rffi.CCHARP.TO, buflen, + flavor='raw', zero=True) py_str.c_size = length py_str.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, space.w_str)) return py_str @@ -61,14 +61,13 @@ pto = rffi.cast(PyObject, pto) Py_DecRef(space, pto) - at cpython_api([CONST_STRING, Py_ssize_t], PyStringObject, error=lltype.nullptr(PyStringObject.TO)) + at cpython_api([CONST_STRING, Py_ssize_t], PyObject) def PyString_FromStringAndSize(space, char_p, length): if char_p: s = rffi.charpsize2str(char_p, length) - ptr = make_ref(space, space.wrap(s)) - return rffi.cast(PyStringObject, ptr) + return make_ref(space, space.wrap(s)) else: - return new_empty_str(space, length) + return rffi.cast(PyObject, new_empty_str(space, length)) @cpython_api([CONST_STRING], PyObject) def PyString_FromString(space, char_p): @@ -125,7 +124,6 @@ to_cp = oldsize for i in range(to_cp): py_newstr.c_buffer[i] = py_str.c_buffer[i] - py_newstr.c_buffer[newsize] = '\x00' Py_DecRef(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Tue Apr 20 14:16:42 2010 @@ -59,7 +59,7 @@ char* c; Py_ssize_t len; - s = PyString_FromStringAndSize(NULL, 3); + s = PyString_FromStringAndSize(NULL, 4); if (s == NULL) return NULL; t = PyString_FromStringAndSize(NULL, 3); @@ -67,16 +67,15 @@ return NULL; Py_DECREF(t); c = PyString_AsString(s); - //len = PyString_Size(s); c[0] = 'a'; - c[1] = 'b'; - c[2] = 'c';//len-1] = 'c'; + c[1] = 'b'; + c[3] = 'c'; return s; """), ]) s = module.getstring() - assert len(s) == 3 - assert s == 'abc' + assert len(s) == 4 + assert s == 'ab\x00c' From arigo at codespeak.net Tue Apr 20 14:36:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 14:36:06 +0200 (CEST) Subject: [pypy-svn] r73905 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100420123606.615D7282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 14:36:04 2010 New Revision: 73905 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Perform three regallocs on each graph: for ints, for refs and for floats. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Tue Apr 20 14:36:04 2010 @@ -35,7 +35,6 @@ # all_funcs = [] for key in self._insns: - assert key is not None, "hole!" assert key.count('/') == 1, "bad key: %r" % (key,) name, argcodes = key.split('/') all_funcs.append(self._get_method(name, argcodes)) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Tue Apr 20 14:36:04 2010 @@ -3,6 +3,8 @@ from pypy.jit.codewriter.flatten import flatten_graph from pypy.jit.codewriter.assembler import Assembler +KINDS = ['int', 'ref', 'float'] + class CodeWriter(object): @@ -16,7 +18,9 @@ return self.transform_graph_to_jitcode(graph) def transform_graph_to_jitcode(self, graph): - regalloc = perform_register_allocation(graph) - ssarepr = flatten_graph(graph, regalloc) + regallocs = {} + for kind in KINDS: + regallocs[kind] = perform_register_allocation(graph, kind) + ssarepr = flatten_graph(graph, regallocs) jitcode = self.assembler.assemble(ssarepr) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 20 14:36:04 2010 @@ -1,4 +1,5 @@ from pypy.objspace.flow.model import Variable +from pypy.jit.metainterp.history import getkind class SSARepr(object): @@ -28,10 +29,10 @@ # ____________________________________________________________ -def flatten_graph(graph, regalloc): +def flatten_graph(graph, regallocs): """Flatten the graph into an SSARepr, with already-computed register - allocations.""" - flattener = GraphFlattener(graph, regalloc) + allocations. 'regallocs' in a dict {kind: RegAlloc}.""" + flattener = GraphFlattener(graph, regallocs) flattener.enforce_input_args() flattener.generate_ssa_form() return flattener.assembler @@ -39,20 +40,24 @@ class GraphFlattener(object): - def __init__(self, graph, regalloc): + def __init__(self, graph, regallocs): self.graph = graph - self.regalloc = regalloc + self.regallocs = regallocs self.registers = {} def enforce_input_args(self): inputargs = self.graph.startblock.inputargs - for i in range(len(inputargs)): - col = self.regalloc.getcolor(inputargs[i]) - if col != i: - assert col > i - self.regalloc.swapcolors(i, col) - for i in range(len(inputargs)): - assert self.regalloc.getcolor(inputargs[i]) == i + numkinds = {} + for v in inputargs: + kind = getkind(v.concretetype) + if kind == 'void': + continue + curcol = self.regallocs[kind].getcolor(v) + realcol = numkinds.get(kind, 0) + numkinds[kind] = realcol + 1 + if curcol != realcol: + assert curcol > realcol + self.regallocs[kind].swapcolors(realcol, curcol) def generate_ssa_form(self): self.assembler = SSARepr(self.graph.name) @@ -162,7 +167,8 @@ self.emitline(op.opname, *args) def getcolor(self, v): - col = self.regalloc.getcolor(v) + kind = getkind(v.concretetype) + col = self.regallocs[kind].getcolor(v) # if kind=='void', fix caller try: r = self.registers[col] except KeyError: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Tue Apr 20 14:36:04 2010 @@ -2,10 +2,12 @@ from pypy.objspace.flow.model import Variable from pypy.tool.algo.color import DependencyGraph from pypy.tool.algo.unionfind import UnionFind +from pypy.jit.metainterp.history import getkind - -def perform_register_allocation(graph): - regalloc = RegAllocator(graph) +def perform_register_allocation(graph, kind): + """Perform register allocation for the Variables of the given 'kind' + in the 'graph'.""" + regalloc = RegAllocator(graph, kind) regalloc.make_dependencies() regalloc.coalesce_variables() regalloc.find_node_coloring() @@ -15,8 +17,9 @@ class RegAllocator(object): DEBUG_REGALLOC = False - def __init__(self, graph): + def __init__(self, graph, kind): self.graph = graph + self.kind = kind def make_dependencies(self): dg = DependencyGraph() @@ -38,20 +41,23 @@ dg.add_node(v) for j in range(i): dg.add_edge(block.inputargs[j], v) - livevars = set(block.inputargs) die_at = [(value, key) for (key, value) in die_at.items()] die_at.sort() die_at.append((sys.maxint,)) + # Done. XXX the code above this line runs 3 times + # (for kind in KINDS) to produce the same result... + livevars = set(block.inputargs) die_index = 0 for i, op in enumerate(block.operations): while die_at[die_index][0] == i: livevars.remove(die_at[die_index][1]) die_index += 1 - if op.result is not None: + if getkind(op.result.concretetype) == self.kind: livevars.add(op.result) dg.add_node(op.result) for v in livevars: - dg.add_edge(v, op.result) + if getkind(v.concretetype) == self.kind: + dg.add_edge(v, op.result) self._depgraph = dg def coalesce_variables(self): @@ -68,7 +74,8 @@ # middle during blackholing. for link in block.exits: for i, v in enumerate(link.args): - if isinstance(v, Variable): + if (isinstance(v, Variable) and + getkind(v.concretetype) == self.kind): w = link.target.inputargs[i] v0 = uf.find_rep(v) w0 = uf.find_rep(w) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 20 14:36:04 2010 @@ -24,7 +24,7 @@ def encoding_test(self, func, args, expected, optimize=True): graphs = self.make_graphs(func, args) - ssarepr = flatten_graph(graphs[0], FakeRegAlloc()) + ssarepr = flatten_graph(graphs[0], {'int': FakeRegAlloc()}) asm = format_assembler(ssarepr) expected = str(py.code.Source(expected)).strip() + '\n' assert asm == expected Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Tue Apr 20 14:36:04 2010 @@ -12,8 +12,11 @@ return self.rtyper.annotator.translator.graphs def check_assembler(self, graph, expected): - regalloc = perform_register_allocation(graph) - ssarepr = flatten_graph(graph, regalloc) + """Only for simple graphs. More complex graphs must first be + transformed by jitter.py before they can be subjected to + register allocation and flattening.""" + regalloc = perform_register_allocation(graph, 'int') + ssarepr = flatten_graph(graph, {'int': regalloc}) asm = format_assembler(ssarepr) assert asm == str(py.code.Source(expected).strip()) + '\n' @@ -21,13 +24,23 @@ def f(a, b): return a + b graph = self.make_graphs(f, [5, 6])[0] - regalloc = perform_register_allocation(graph) + regalloc = perform_register_allocation(graph, 'int') va, vb = graph.startblock.inputargs vc = graph.startblock.operations[0].result assert regalloc.getcolor(va) == 0 assert regalloc.getcolor(vb) == 1 assert regalloc.getcolor(vc) == 0 + def test_regalloc_void(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + graph = self.make_graphs(f, [5, 6])[0] + regalloc = perform_register_allocation(graph, 'float') + # assert did not crash + def test_regalloc_loop(self): def f(a, b): while a > 0: From arigo at codespeak.net Tue Apr 20 14:51:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 14:51:50 +0200 (CEST) Subject: [pypy-svn] r73906 - in pypy/branch/blackhole-improvement/pypy/tool/algo: . test Message-ID: <20100420125150.E2F94282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 14:51:49 2010 New Revision: 73906 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Support empty DependencyGraphs. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Tue Apr 20 14:51:49 2010 @@ -29,6 +29,8 @@ def lexicographic_order(self): """Enumerate a lexicographic breath-first ordering of the nodes.""" sigma = [self.getnodes()[::-1]] + if not sigma[0]: + return while sigma: v = sigma[0].pop() yield v Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Tue Apr 20 14:51:49 2010 @@ -27,6 +27,11 @@ # of insertion of nodes. assert ''.join(order) == 'acbde' +def test_lexicographic_order_empty(): + dg = DependencyGraph() + order = list(dg.lexicographic_order()) + assert order == [] + def test_size_of_largest_clique(): dg = graph1() assert dg.size_of_largest_clique() == 3 @@ -40,3 +45,8 @@ for v1, v2list in dg.neighbours.items(): for v2 in v2list: assert coloring[v1] != coloring[v2] + +def test_find_node_coloring_empty(): + dg = DependencyGraph() + coloring = dg.find_node_coloring() + assert coloring == {} From arigo at codespeak.net Tue Apr 20 14:56:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 14:56:51 +0200 (CEST) Subject: [pypy-svn] r73907 - pypy/branch/blackhole-improvement/pypy/tool/algo/test Message-ID: <20100420125651.42F86282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 14:56:49 2010 New Revision: 73907 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Log: Fix test. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/test/test_color.py Tue Apr 20 14:56:49 2010 @@ -8,9 +8,9 @@ dg.add_node('c') dg.add_node('d') dg.add_node('e') - dg.add_edge('a', 'b') - dg.add_edge('a', 'd') - dg.add_edge('d', 'b') + dg.add_edge('a', 'b') # d---e + dg.add_edge('a', 'd') # / \ / \ + dg.add_edge('d', 'b') # a---b---c dg.add_edge('d', 'e') dg.add_edge('b', 'c') dg.add_edge('b', 'e') @@ -21,11 +21,10 @@ dg = graph1() order = list(dg.lexicographic_order()) assert len(order) == 5 - order.reverse() # there are many orders that are correct answers, but check that we get # the following one, which is somehow the 'first' answer in the order # of insertion of nodes. - assert ''.join(order) == 'acbde' + assert ''.join(order) == 'abdec' def test_lexicographic_order_empty(): dg = DependencyGraph() From arigo at codespeak.net Tue Apr 20 14:57:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 14:57:25 +0200 (CEST) Subject: [pypy-svn] r73908 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100420125725.EDAAE282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 14:57:24 2010 New Revision: 73908 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Log: Fix. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Tue Apr 20 14:57:24 2010 @@ -36,21 +36,26 @@ for link in block.exits: for v in link.args: die_at.pop(v, None) - # Add the variables of this block to the dependency graph - for i, v in enumerate(block.inputargs): - dg.add_node(v) - for j in range(i): - dg.add_edge(block.inputargs[j], v) die_at = [(value, key) for (key, value) in die_at.items()] die_at.sort() die_at.append((sys.maxint,)) # Done. XXX the code above this line runs 3 times # (for kind in KINDS) to produce the same result... - livevars = set(block.inputargs) + livevars = [v for v in block.inputargs + if getkind(v.concretetype) == self.kind] + # Add the variables of this block to the dependency graph + for i, v in enumerate(livevars): + dg.add_node(v) + for j in range(i): + dg.add_edge(livevars[j], v) + livevars = set(livevars) die_index = 0 for i, op in enumerate(block.operations): while die_at[die_index][0] == i: - livevars.remove(die_at[die_index][1]) + try: + livevars.remove(die_at[die_index][1]) + except KeyError: + pass die_index += 1 if getkind(op.result.concretetype) == self.kind: livevars.add(op.result) From arigo at codespeak.net Tue Apr 20 15:04:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 15:04:41 +0200 (CEST) Subject: [pypy-svn] r73909 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100420130441.C7D72282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 15:04:40 2010 New Revision: 73909 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Log: Store the kind on each Register. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 20 15:04:40 2010 @@ -24,7 +24,8 @@ return isinstance(other, TLabel) and other.name == self.name class Register(object): - def __init__(self, index): + def __init__(self, kind, index): + self.kind = kind self.index = index # ____________________________________________________________ @@ -170,7 +171,7 @@ kind = getkind(v.concretetype) col = self.regallocs[kind].getcolor(v) # if kind=='void', fix caller try: - r = self.registers[col] + r = self.registers[kind, col] except KeyError: - r = self.registers[col] = Register(col) + r = self.registers[kind, col] = Register(kind, col) return r Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Tue Apr 20 15:04:40 2010 @@ -10,7 +10,7 @@ # def repr(x): if isinstance(x, Register): - return '%i' + str(x.index) + return '%%%s%d' % (x.kind[0], x.index) # e.g. %i1 or %r2 or %f3 elif isinstance(x, Constant): return '$' + str(x.value) elif isinstance(x, TLabel): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Tue Apr 20 15:04:40 2010 @@ -6,7 +6,7 @@ def test_assemble_simple(): ssarepr = SSARepr("test") - i0, i1, i2 = Register(0), Register(1), Register(2) + i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) ssarepr.insns = [ ('int_add', i0, i1, i2), ('int_return', i2), @@ -21,7 +21,7 @@ def test_assemble_consts(): ssarepr = SSARepr("test") ssarepr.insns = [ - ('int_return', Register(13)), + ('int_return', Register('int', 13)), ('int_return', Constant(18)), ('int_return', Constant(-4)), ('int_return', Constant(128)), @@ -40,7 +40,7 @@ def test_assemble_loop(): ssarepr = SSARepr("test") - i0, i1 = Register(0x16), Register(0x17) + i0, i1 = Register('int', 0x16), Register('int', 0x17) ssarepr.insns = [ (Label('L1'),), ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4)), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Tue Apr 20 15:04:40 2010 @@ -6,7 +6,7 @@ def test_format_assembler_simple(): ssarepr = SSARepr("test") - i0, i1, i2 = Register(0), Register(1), Register(2) + i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) ssarepr.insns = [ ('foobar', [i0, i1]), ('int_add', i0, i1, i2), @@ -20,9 +20,21 @@ """ assert asm == str(py.code.Source(expected)).strip() + '\n' +def test_format_assembler_float(): + ssarepr = SSARepr("test") + i1, r2, f3 = Register('int', 1), Register('ref', 2), Register('float', 3) + ssarepr.insns = [ + ('foobar', i1, r2, f3), + ] + asm = format_assembler(ssarepr) + expected = """ + foobar %i1, %r2, %f3 + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' + def test_format_assembler_loop(): ssarepr = SSARepr("test") - i0, i1 = Register(0), Register(1) + i0, i1 = Register('int', 0), Register('int', 1) ssarepr.insns = [ ('foobar', [i0, i1]), (Label('L1'),), From arigo at codespeak.net Tue Apr 20 15:42:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Apr 2010 15:42:49 +0200 (CEST) Subject: [pypy-svn] r73910 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100420134249.83F3B282BD4@codespeak.net> Author: arigo Date: Tue Apr 20 15:42:47 2010 New Revision: 73910 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Fix the handling of constants in the BlackholeInterpreter. More general fixes here and there to support refs and floats. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Tue Apr 20 15:42:47 2010 @@ -1,11 +1,12 @@ -from pypy.jit.metainterp import history -from pypy.jit.codewriter.flatten import Register, Label, TLabel +from pypy.jit.metainterp.history import AbstractValue, getkind +from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS from pypy.objspace.flow.model import Constant -from pypy.jit.metainterp.history import ConstInt -class JitCode(history.AbstractValue): - empty_list = [] +class JitCode(AbstractValue): + _empty_i = [] + _empty_r = [] + _empty_f = [] def __init__(self, name, cfnptr=None, calldescr=None, called_from=None, graph=None): @@ -15,9 +16,12 @@ self.called_from = called_from self.graph = graph - def setup(self, code, constants): + def setup(self, code, constants_i=[], constants_r=[], constants_f=[]): self.code = code - self.constants = constants or self.empty_list # share the empty list + # if the following lists are empty, use a single shared empty list + self.constants_i = constants_i or self._empty_i + self.constants_r = constants_r or self._empty_r + self.constants_f = constants_f or self._empty_f class Assembler(object): @@ -28,9 +32,12 @@ def assemble(self, ssarepr): code = [] constants_dict = {} - constants_rev = [] + constants_i = [] + constants_r = [] + constants_f = [] label_positions = {} tlabel_positions = [] + highest_regs = dict.fromkeys(KINDS, 0) for insn in ssarepr.insns: if isinstance(insn[0], Label): label_positions[insn[0].name] = len(code) @@ -41,18 +48,29 @@ argcodes = [] for x in insn[1:]: if isinstance(x, Register): + if x.index > highest_regs[x.kind]: + highest_regs[x.kind] = x.index code.append(chr(x.index)) - argcodes.append('i') + argcodes.append(x.kind[0]) elif isinstance(x, Constant): - if -128 <= x.value <= 127: + kind = getkind(x.concretetype) + if kind == 'int' and -128 <= x.value <= 127: code.append(chr(x.value & 0xFF)) argcodes.append('c') else: if x not in constants_dict: - constants_rev.append(ConstInt(x.value)) - constants_dict[x] = 256 - len(constants_rev) + if kind == 'int': + constants = constants_i + elif kind == 'ref': + constants = constants_r + elif kind == 'float': + constants = constants_f + else: + raise NotImplementedError(x) + constants.append(x.value) + constants_dict[x] = 256 - len(constants) code.append(chr(constants_dict[x])) - argcodes.append('i') + argcodes.append(kind[0]) elif isinstance(x, TLabel): tlabel_positions.append((x.name, len(code))) code.append("temp 1") @@ -73,6 +91,11 @@ code[pos ] = chr(target & 0xFF) code[pos+1] = chr(target >> 8) # + # Limitation of the number of registers, from the single-byte encoding + assert highest_regs['int'] + len(constants_i) <= 256 + assert highest_regs['ref'] + len(constants_r) <= 256 + assert highest_regs['float'] + len(constants_f) <= 256 + # jitcode = JitCode(ssarepr.name) - jitcode.setup(''.join(code), constants_rev[::-1]) + jitcode.setup(''.join(code), constants_i, constants_r, constants_f) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Tue Apr 20 15:42:47 2010 @@ -15,6 +15,9 @@ class LeaveFrame(Exception): pass +class MissingValue(object): + pass + def signedord(c): value = ord(c) value = intmask(value << (LONG_BIT-8)) >> (LONG_BIT-8) @@ -24,7 +27,15 @@ class BlackholeInterpreter(object): def __init__(self): + self.registers_i = [MissingValue()] * 256 + self.registers_r = [MissingValue()] * 256 + self.registers_f = [MissingValue()] * 256 + + def _freeze_(self): self.registers_i = [0] * 256 + self.registers_r = [NULL] * 256 + self.registers_f = [0.0] * 256 + return False def setup_insns(self, insns): assert len(insns) <= 256, "too many instructions!" @@ -106,13 +117,24 @@ self.registers_i[index] = value def run(self, jitcode, position): + self.copy_constants(self.registers_i, jitcode.constants_i) + self.copy_constants(self.registers_r, jitcode.constants_r) + self.copy_constants(self.registers_f, jitcode.constants_f) code = jitcode.code - constants = jitcode.constants try: self.dispatch_loop(code, position) except LeaveFrame: pass + # XXX must be specialized + def copy_constants(self, registers, constants): + i = len(constants) + while i > 0: + j = 256 - i + assert j >= 0 + i -= 1 + registers[j] = constants[i] + # ---------- @arguments("i", "i", returns="i") Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Tue Apr 20 15:42:47 2010 @@ -1,10 +1,8 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.regalloc import perform_register_allocation -from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler -KINDS = ['int', 'ref', 'float'] - class CodeWriter(object): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 20 15:42:47 2010 @@ -28,6 +28,8 @@ self.kind = kind self.index = index +KINDS = ['int', 'ref', 'float'] + # ____________________________________________________________ def flatten_graph(graph, regallocs): @@ -85,7 +87,12 @@ def make_return(self, args): if len(args) == 1: # return from function - self.emitline("int_return", self.getcolor(args[0])) + [v] = args + kind = getkind(v.concretetype) + if kind == 'void': + self.emitline("void_return") + else: + self.emitline("%s_return" % kind, self.getcolor(args[0])) elif len(args) == 2: # exception block, raising an exception from a function xxx Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Tue Apr 20 15:42:47 2010 @@ -1,7 +1,7 @@ from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register from pypy.objspace.flow.model import Constant -from pypy.jit.metainterp.history import ConstInt +from pypy.rpython.lltypesystem import lltype def test_assemble_simple(): @@ -22,10 +22,10 @@ ssarepr = SSARepr("test") ssarepr.insns = [ ('int_return', Register('int', 13)), - ('int_return', Constant(18)), - ('int_return', Constant(-4)), - ('int_return', Constant(128)), - ('int_return', Constant(-129)), + ('int_return', Constant(18, lltype.Signed)), + ('int_return', Constant(-4, lltype.Signed)), + ('int_return', Constant(128, lltype.Signed)), + ('int_return', Constant(-129, lltype.Signed)), ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) @@ -36,16 +36,33 @@ "\x00\xFE") assert assembler.insns == {'int_return/i': 0, 'int_return/c': 1} - assert jitcode.constants == [ConstInt(-129), ConstInt(128)] + assert jitcode.constants_i == [128, -129] + +def test_assemble_float_consts(): + ssarepr = SSARepr("test") + ssarepr.insns = [ + ('float_return', Register('float', 13)), + ('float_return', Constant(18.0, lltype.Float)), + ('float_return', Constant(-4.0, lltype.Float)), + ('float_return', Constant(128.1, lltype.Float)), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x0D" + "\x00\xFF" + "\x00\xFE" + "\x00\xFD") + assert assembler.insns == {'float_return/f': 0} + assert jitcode.constants_f == [18.0, -4.0, 128.1] def test_assemble_loop(): ssarepr = SSARepr("test") i0, i1 = Register('int', 0x16), Register('int', 0x17) ssarepr.insns = [ (Label('L1'),), - ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4)), + ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(4, lltype.Signed)), ('int_add', i1, i0, i1), - ('int_sub', i0, Constant(1), i0), + ('int_sub', i0, Constant(1, lltype.Signed), i0), ('goto', TLabel('L1')), (Label('L2'),), ('int_return', i1), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Tue Apr 20 15:42:47 2010 @@ -27,6 +27,18 @@ blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 42 +def test_simple_bigconst(): + jitcode = JitCode("test") + jitcode.setup("\x00\xFD\x01\x02" + "\x01\x02", + [666, 666, 10042, 666]) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_sub/iii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(1, 10000) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + def test_simple_loop(): jitcode = JitCode("test") jitcode.setup("\x00\x10\x00\x16\x02" # L1: goto_if_not_int_gt L2, %i0, 2 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 20 15:42:47 2010 @@ -24,7 +24,9 @@ def encoding_test(self, func, args, expected, optimize=True): graphs = self.make_graphs(func, args) - ssarepr = flatten_graph(graphs[0], {'int': FakeRegAlloc()}) + ssarepr = flatten_graph(graphs[0], {'int': FakeRegAlloc(), + 'ref': FakeRegAlloc(), + 'float': FakeRegAlloc()}) asm = format_assembler(ssarepr) expected = str(py.code.Source(expected)).strip() + '\n' assert asm == expected @@ -56,3 +58,14 @@ L2: int_return %i3 """) + + def test_float(self): + def f(i, f): + return (i*5) + (f*0.25) + self.encoding_test(f, [4, 7.5], """ + int_mul %i0, $5, %i1 + float_mul %f0, $0.25, %f1 + cast_int_to_float %i1, %f2 + float_add %f2, %f1, %f3 + float_return %f3 + """) From jandem at codespeak.net Tue Apr 20 16:27:12 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Tue, 20 Apr 2010 16:27:12 +0200 (CEST) Subject: [pypy-svn] r73911 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420142712.ABBF4282BD4@codespeak.net> Author: jandem Date: Tue Apr 20 16:27:11 2010 New Revision: 73911 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py Log: Add wrappers for __setitem__, __delitem__ and slices. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Tue Apr 20 16:27:11 2010 @@ -5,7 +5,7 @@ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc,\ - ssizeargfunc + ssizeargfunc, ssizessizeargfunc, ssizeobjargproc from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt @@ -62,6 +62,29 @@ index = space.int_w(space.index(args_w[0])) return generic_cpy_call(space, func_target, w_self, index) +def wrap_sq_setitem(space, w_self, w_args, func): + func_target = rffi.cast(ssizeobjargproc, func) + check_num_args(space, w_args, 2) + args_w = space.fixedview(w_args) + index = space.int_w(space.index(args_w[0])) + return generic_cpy_call(space, func_target, w_self, index, args_w[1]) + +def wrap_sq_delitem(space, w_self, w_args, func): + func_target = rffi.cast(ssizeobjargproc, func) + check_num_args(space, w_args, 1) + args_w = space.fixedview(w_args) + index = space.int_w(space.index(args_w[0])) + null = lltype.nullptr(PyObject.TO) + return generic_cpy_call(space, func_target, w_self, index, null) + +def wrap_ssizessizeargfunc(space, w_self, w_args, func): + func_target = rffi.cast(ssizessizeargfunc, func) + check_num_args(space, w_args, 2) + args_w = space.fixedview(w_args) + start = space.int_w(args_w[0]) + end = space.int_w(args_w[1]) + return generic_cpy_call(space, func_target, w_self, start, end) + @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_arraymodule.py Tue Apr 20 16:27:11 2010 @@ -14,3 +14,21 @@ arr.append(4) assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + + def test_index(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert arr[3] == 4 + raises(IndexError, arr.__getitem__, 10) + del arr[2] + assert arr.tolist() == [1,2,4] + arr[2] = 99 + assert arr.tolist() == [1,2,99] + + def test_slice_get(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert arr[:].tolist() == [1,2,3,4] + assert arr[1:].tolist() == [2,3,4] + assert arr[:2].tolist() == [1,2] + assert arr[1:3].tolist() == [2,3] From afa at codespeak.net Tue Apr 20 17:16:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 17:16:15 +0200 (CEST) Subject: [pypy-svn] r73912 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420151615.4133E282BD4@codespeak.net> Author: afa Date: Tue Apr 20 17:16:13 2010 New Revision: 73912 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Add a few forward declarations like "struct PyIntObject;" to plase boost::python. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 20 17:16:13 2010 @@ -255,27 +255,38 @@ '_Py_ZeroStruct#': ('PyObject*', 'space.w_False'), '_Py_NotImplementedStruct#': ('PyObject*', 'space.w_NotImplemented'), } +FORWARD_DECLS = [] INIT_FUNCTIONS = [] BOOTSTRAP_FUNCTIONS = [] -for exc_name in exceptions.Module.interpleveldefs.keys(): - GLOBALS['PyExc_' + exc_name] = ('PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) - -for cpyname, pypyexpr in {"Type": "space.w_type", - "BaseObject": "space.w_object", +def build_exported_objects(): + # Standard exceptions + for exc_name in exceptions.Module.interpleveldefs.keys(): + GLOBALS['PyExc_' + exc_name] = ( + 'PyTypeObject*', + 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) + + # Common types with their own struct + for cpyname, pypyexpr in { + "Type": "space.w_type", + "String": "space.w_str", + "Unicode": "space.w_unicode", "Dict": "space.w_dict", "Tuple": "space.w_tuple", "List": "space.w_list", - "String": "space.w_str", - "Unicode": "space.w_unicode", "Int": "space.w_int", "Bool": "space.w_bool", "Float": "space.w_float", "Long": "space.w_long", + "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', }.items(): - GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) + GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) + + for cpyname in 'Method List Int Long Dict Tuple'.split(): + FORWARD_DECLS.append('struct Py%sObject' % (cpyname, )) +build_exported_objects() def get_structtype_for_ctype(ctype): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr @@ -564,6 +575,10 @@ functions = [] pypy_decls = [] pypy_decls.append("#ifndef PYPY_STANDALONE\n") + + for decl in FORWARD_DECLS: + pypy_decls.append("%s;" % (decl,)) + for name, func in sorted(FUNCTIONS.iteritems()): restype = db.gettype(func.restype).replace('@', '').strip() args = [] @@ -597,7 +612,7 @@ pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) if not globals_are_pointers and "#" not in name: pypy_decls.append("#define %s &%s" % (name, name,)) - pypy_decls.append("#endif\n") + pypy_decls.append("#endif /*PYPY_STANDALONE*/\n") pypy_decl_h = udir.join('pypy_decl.h') pypy_decl_h.write('\n'.join(pypy_decls)) From afa at codespeak.net Tue Apr 20 17:53:40 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 17:53:40 +0200 (CEST) Subject: [pypy-svn] r73915 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420155340.AEA6E282BD4@codespeak.net> Author: afa Date: Tue Apr 20 17:53:38 2010 New Revision: 73915 Added: pypy/branch/cpython-extension/pypy/module/cpyext/complexobject.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_complexobject.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Some part of PyComplex object. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Tue Apr 20 17:53:38 2010 @@ -65,6 +65,7 @@ import pypy.module.cpyext.stubsactive import pypy.module.cpyext.pystate import pypy.module.cpyext.datetime +import pypy.module.cpyext.complexobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 20 17:53:38 2010 @@ -278,6 +278,7 @@ "Bool": "space.w_bool", "Float": "space.w_float", "Long": "space.w_long", + "Complex": "space.w_complex", "BaseObject": "space.w_object", 'None': 'space.type(space.w_None)', 'NotImplemented': 'space.type(space.w_NotImplemented)', Added: pypy/branch/cpython-extension/pypy/module/cpyext/complexobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/complexobject.py Tue Apr 20 17:53:38 2010 @@ -0,0 +1,27 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers +from pypy.objspace.std.complexobject import W_ComplexObject + +PyComplex_Check, PyComplex_CheckExact = build_type_checkers("Complex") + + at cpython_api([lltype.Float, lltype.Float], PyObject) +def PyComplex_FromDoubles(space, real, imag): + return space.newcomplex(real, imag) + + at cpython_api([PyObject], lltype.Float, error=-1) +def PyComplex_RealAsDouble(space, w_obj): + if space.is_true(space.isinstance(w_obj, space.w_complex)): + assert isinstance(w_obj, W_ComplexObject) + return w_obj.realval + else: + return space.float_w(w_obj) + + at cpython_api([PyObject], lltype.Float, error=-1) +def PyComplex_ImagAsDouble(space, w_obj): + if space.is_true(space.isinstance(w_obj, space.w_complex)): + assert isinstance(w_obj, W_ComplexObject) + return w_obj.imagval + else: + # CPython also accepts anything + return 0.0 + Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_complexobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_complexobject.py Tue Apr 20 17:53:38 2010 @@ -0,0 +1,19 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestComplexObject(BaseApiTest): + def test_complexobject(self, space, api): + w_value = api.PyComplex_FromDoubles(1.2, 3.4) + assert space.unwrap(w_value) == 1.2+3.4j + assert api.PyComplex_RealAsDouble(w_value) == 1.2 + assert api.PyComplex_ImagAsDouble(w_value) == 3.4 + + assert api.PyComplex_RealAsDouble(space.wrap(42)) == 42 + assert api.PyComplex_RealAsDouble(space.wrap(1.5)) == 1.5 + assert api.PyComplex_ImagAsDouble(space.wrap(1.5)) == 0.0 + + # cpython accepts anything for PyComplex_ImagAsDouble + assert api.PyComplex_ImagAsDouble(space.w_None) == 0.0 + assert not api.PyErr_Occurred() + assert api.PyComplex_RealAsDouble(space.w_None) == -1.0 + assert api.PyErr_Occurred() + api.PyErr_Clear() From afa at codespeak.net Tue Apr 20 18:58:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 18:58:44 +0200 (CEST) Subject: [pypy-svn] r73916 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420165844.BCFD0282BD6@codespeak.net> Author: afa Date: Tue Apr 20 18:58:42 2010 New Revision: 73916 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: implement PyUnicode_FromWideChar and PyUnicode_AsWideChar. unlike other functions, PyUnicode_AsWideChar takes a PyUnicodeObject as first argument. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 20 18:58:42 2010 @@ -56,8 +56,12 @@ _compilation_info_ = CConfig._compilation_info_ VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list') -CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) +CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char, + hints={'nolength': True})) +CONST_WSTRING = lltype.Ptr(lltype.Array(lltype.UniChar, + hints={'nolength': True})) assert CONST_STRING is not rffi.CCHARP +assert CONST_WSTRING is not rffi.CWCHARP constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING @@ -586,6 +590,8 @@ for i, argtype in enumerate(func.argtypes): if argtype is CONST_STRING: arg = 'const char *@' + elif argtype is CONST_WSTRING: + arg = 'const wchar_t *@' else: arg = db.gettype(argtype) arg = arg.replace('@', 'arg%d' % (i,)).strip() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 20 18:58:42 2010 @@ -5626,30 +5626,6 @@ throughout the interpreter whenever coercion to Unicode is needed.""" raise NotImplementedError - at cpython_api([{const wchar_t*}, Py_ssize_t], PyObject) -def PyUnicode_FromWideChar(space, w, size): - """Create a Unicode object from the wchar_t buffer w of the given size. - Return NULL on failure. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([{PyUnicodeObject*}, {wchar_t*}, Py_ssize_t], Py_ssize_t) -def PyUnicode_AsWideChar(space, unicode, w, size): - """Copy the Unicode object contents into the wchar_t buffer w. At most - size wchar_t characters are copied (excluding a possibly trailing - 0-termination character). Return the number of wchar_t characters - copied or -1 in case of an error. Note that the resulting wchar_t - string may or may not be 0-terminated. It is the responsibility of the caller - to make sure that the wchar_t string is 0-terminated in case this is - required by the application. - - This function returned an int type and used an int - type for size. This might require changes in your code for properly - supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) def PyUnicode_Decode(space, s, size, encoding, errors): """Create a Unicode object by decoding size bytes of the encoded string s. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Tue Apr 20 18:58:42 2010 @@ -49,6 +49,13 @@ space.wrap(''), None, None) rffi.free_charp(utf_8) + buf = rffi.unicode2wcharp(u"12345") + api.PyUnicode_AsWideChar(space.wrap(u'longword'), buf, 5) + assert rffi.wcharp2unicode(buf) == 'longw' + api.PyUnicode_AsWideChar(space.wrap(u'a'), buf, 5) + assert rffi.wcharp2unicode(buf) == 'a' + lltype.free(buf, flavor='raw') + def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 20 18:58:42 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.api import ( CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, bootstrap_function, generic_cpy_call, PyObjectFields, - cpython_struct, CONST_STRING) + cpython_struct, CONST_STRING, CONST_WSTRING) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef, make_typedescr from pypy.module.cpyext.stringobject import PyString_Check @@ -142,6 +142,37 @@ space.wrap("expected unicode object")) return PyUnicode_AS_UNICODE(space, ref) + at cpython_api([PyUnicodeObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1) +def PyUnicode_AsWideChar(space, ref, buf, size): + """Copy the Unicode object contents into the wchar_t buffer w. At most + size wchar_t characters are copied (excluding a possibly trailing + 0-termination character). Return the number of wchar_t characters + copied or -1 in case of an error. Note that the resulting wchar_t + string may or may not be 0-terminated. It is the responsibility of the caller + to make sure that the wchar_t string is 0-terminated in case this is + required by the application.""" + if not PyUnicode_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected unicode object")) + + c_buffer = PyUnicode_AS_UNICODE(space, ref) + c_size = rffi.cast(PyUnicodeObject, ref).c_size + + # If possible, try to copy the 0-termination as well + if size > c_size: + size = c_size + 1 + + + i = 0 + while i < size: + buf[i] = c_buffer[i] + i += 1 + + if size > c_size: + return c_size + else: + return size + @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def PyUnicode_GetDefaultEncoding(space): """Returns the currently active default encoding.""" @@ -179,7 +210,7 @@ w_errors = rffi.charp2str(encoding) return unicodetype.encode_object(space, w_unicode, w_encoding, w_errors) - at cpython_api([rffi.CWCHARP, Py_ssize_t], PyObject) + at cpython_api([CONST_WSTRING, Py_ssize_t], PyObject) def PyUnicode_FromUnicode(space, wchar_p, length): """Create a Unicode Object from the Py_UNICODE buffer u of the given size. u may be NULL which causes the contents to be undefined. It is the user's @@ -193,6 +224,13 @@ ptr = make_ref(space, space.wrap(s)) return ptr + at cpython_api([CONST_WSTRING, Py_ssize_t], PyObject) +def PyUnicode_FromWideChar(space, wchar_p, length): + """Create a Unicode object from the wchar_t buffer w of the given size. + Return NULL on failure.""" + # PyPy supposes Py_UNICODE == wchar_t + return PyUnicode_FromUnicode(space, wchar_p, length) + @cpython_api([PyObject, CONST_STRING], PyObject) def _PyUnicode_AsDefaultEncodedString(space, w_unicode, errors): return PyUnicode_AsEncodedString(space, w_unicode, lltype.nullptr(rffi.CCHARP.TO), errors) From afa at codespeak.net Tue Apr 20 19:09:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 19:09:23 +0200 (CEST) Subject: [pypy-svn] r73917 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420170923.C8920282BD6@codespeak.net> Author: afa Date: Tue Apr 20 19:09:22 2010 New Revision: 73917 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: fix translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Tue Apr 20 19:09:22 2010 @@ -151,12 +151,8 @@ string may or may not be 0-terminated. It is the responsibility of the caller to make sure that the wchar_t string is 0-terminated in case this is required by the application.""" - if not PyUnicode_Check(space, ref): - raise OperationError(space.w_TypeError, - space.wrap("expected unicode object")) - - c_buffer = PyUnicode_AS_UNICODE(space, ref) - c_size = rffi.cast(PyUnicodeObject, ref).c_size + c_buffer = PyUnicode_AS_UNICODE(space, rffi.cast(PyObject, ref)) + c_size = ref.c_size # If possible, try to copy the 0-termination as well if size > c_size: From tobami at codespeak.net Tue Apr 20 19:15:59 2010 From: tobami at codespeak.net (tobami at codespeak.net) Date: Tue, 20 Apr 2010 19:15:59 +0200 (CEST) Subject: [pypy-svn] r73918 - in pypy/benchmarks: . test Message-ID: <20100420171559.C74E2282BD4@codespeak.net> Author: tobami Date: Tue Apr 20 19:15:58 2010 New Revision: 73918 Modified: pypy/benchmarks/saveresults.py pypy/benchmarks/test/test_saveresults.py Log: changes to acomodate the new codespeed DB Schema Modified: pypy/benchmarks/saveresults.py ============================================================================== --- pypy/benchmarks/saveresults.py (original) +++ pypy/benchmarks/saveresults.py Tue Apr 20 19:15:58 2010 @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +####################################################### +# This script saves result data # +# It expects the format of unladen swallow's perf.py # +####################################################### import urllib, urllib2 from datetime import datetime @@ -10,9 +14,7 @@ #Parse data data = {} current_date = datetime.today() - if branch == "": - print("ERROR: No branch defined") - return 1 + if branch == "": branch = 'trunk' for b in results: bench_name = b[0] @@ -27,16 +29,18 @@ print("ERROR: result type unknown " + b[1]) return 1 data = { - 'revision_number': revision, - 'revision_project': project, - 'revision_branch': branch, - 'interpreter_name': interpreter, - 'interpreter_coptions': int_options, - 'benchmark_name': bench_name, + 'commitid': revision, + 'project': project, + 'branch': branch, + 'executable_name': interpreter, + 'executable_coptions': int_options, + 'benchmark': bench_name, 'environment': host, 'result_value': value, 'result_date': current_date, } + if res_type == "ComparisonResult": + data['std_dev'] = results['std_changed'] if testing: testparams.append(data) else: send(data) if testing: return testparams @@ -47,8 +51,8 @@ params = urllib.urlencode(data) f = None response = "None" - info = str(datetime.today()) + ": Saving result for " + data['interpreter_name'] + " revision " - info += str(data['revision_number']) + ", benchmark " + data['benchmark_name'] + info = str(datetime.today()) + ": Saving result for " + data['executable_name'] + " revision " + info += str(data['commitid']) + ", benchmark " + data['benchmark'] print(info) try: f = urllib2.urlopen(SPEEDURL + 'result/add/', params) @@ -62,4 +66,7 @@ response = '\n The server couldn\'t fulfill the request\n' response += ' Error code: ' + str(e) print("Server (%s) response: %s\n" % (SPEEDURL, response)) - return 1 + return 1 + print "saved correctly!\n" + return 0 + Modified: pypy/benchmarks/test/test_saveresults.py ============================================================================== --- pypy/benchmarks/test/test_saveresults.py (original) +++ pypy/benchmarks/test/test_saveresults.py Tue Apr 20 19:15:58 2010 @@ -23,16 +23,16 @@ def test_good_input(self): '''Given correct result data, check that every result being saved has the right parameters''' for resultparams in saveresults.save("pypy", 71212, self.fixture, "", "experimental", "pypy-c-jit", "gc=hybrid", 'host', True): - assert resultparams['revision_project'] == "pypy" - assert resultparams['revision_number'] == 71212 - assert resultparams['revision_branch'] == "experimental" - assert resultparams['interpreter_name'] == "pypy-c-jit" - assert resultparams['interpreter_coptions'] == "gc=hybrid" + assert resultparams['project'] == "pypy" + assert resultparams['commitid'] == 71212 + assert resultparams['branch'] == "experimental" + assert resultparams['executable_name'] == "pypy-c-jit" + assert resultparams['executable_coptions'] == "gc=hybrid" # get dict with correct data for this benchmark fixturedata = [] benchfound = False for res in self.fixture: - if res[0] == resultparams['benchmark_name']: + if res[0] == resultparams['benchmark']: fixturedata = res benchfound = True break From afa at codespeak.net Tue Apr 20 19:19:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 19:19:04 +0200 (CEST) Subject: [pypy-svn] r73919 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100420171904.3D70D282BD4@codespeak.net> Author: afa Date: Tue Apr 20 19:19:02 2010 New Revision: 73919 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: CPython also casts arguments to Py_INCREF Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Tue Apr 20 19:19:02 2010 @@ -37,10 +37,10 @@ PyObject_VAR_HEAD } PyVarObject; -#define Py_INCREF(ob) (Py_IncRef(ob)) -#define Py_DECREF(ob) (Py_DecRef(ob)) -#define Py_XINCREF(ob) (Py_IncRef(ob)) -#define Py_XDECREF(ob) (Py_DecRef(ob)) +#define Py_INCREF(ob) (Py_IncRef((PyObject *)ob)) +#define Py_DECREF(ob) (Py_DecRef((PyObject *)ob)) +#define Py_XINCREF(ob) (Py_IncRef((PyObject *)ob)) +#define Py_XDECREF(ob) (Py_DecRef((PyObject *)ob)) #define Py_CLEAR(op) \ do { \ From benjamin at codespeak.net Tue Apr 20 22:55:43 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 20 Apr 2010 22:55:43 +0200 (CEST) Subject: [pypy-svn] r73920 - pypy/build/bot2/pypybuildbot Message-ID: <20100420205543.259F6282BD4@codespeak.net> Author: benjamin Date: Tue Apr 20 22:55:41 2010 New Revision: 73920 Modified: pypy/build/bot2/pypybuildbot/master.py Log: applevel64 category Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Tue Apr 20 22:55:41 2010 @@ -175,7 +175,7 @@ "slavenames": ["tannit64"], "builddir": APPLVLLINUX64, "factory": pypyTranslatedAppLevelTestFactory, - "category": "applevel" + "category": "applevel64" }, {"name": STACKLESSAPPLVLLINUX32, "slavenames": ["bigdogvm1", "tannit32"], From afa at codespeak.net Tue Apr 20 23:46:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Apr 2010 23:46:38 +0200 (CEST) Subject: [pypy-svn] r73921 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420214638.74582282BD4@codespeak.net> Author: afa Date: Tue Apr 20 23:46:36 2010 New Revision: 73921 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/state.py Log: The slots setitem and delitem don't return a real value, only an error indicator. This should fix translation. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Tue Apr 20 23:46:36 2010 @@ -67,7 +67,8 @@ check_num_args(space, w_args, 2) args_w = space.fixedview(w_args) index = space.int_w(space.index(args_w[0])) - return generic_cpy_call(space, func_target, w_self, index, args_w[1]) + if generic_cpy_call(space, func_target, w_self, index, args_w[1]) == -1: + space.fromcache(State).check_and_raise_exception() def wrap_sq_delitem(space, w_self, w_args, func): func_target = rffi.cast(ssizeobjargproc, func) @@ -75,7 +76,8 @@ args_w = space.fixedview(w_args) index = space.int_w(space.index(args_w[0])) null = lltype.nullptr(PyObject.TO) - return generic_cpy_call(space, func_target, w_self, index, null) + if generic_cpy_call(space, func_target, w_self, index, null) == -1: + space.fromcache(State).check_and_raise_exception(always=True) def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/state.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/state.py Tue Apr 20 23:46:36 2010 @@ -51,13 +51,16 @@ self.exc_type = None self.exc_value = None - def check_and_raise_exception(self): + def check_and_raise_exception(self, always=False): exc_value = self.exc_value exc_type = self.exc_type if exc_type is not None or exc_value is not None: self.clear_exception() op_err = OperationError(exc_type, exc_value) raise op_err + if always: + raise OperationError(space.w_SystemError, space.wrap( + "Function returned an error result without setting an exception")) def print_refcounts(self): print "REFCOUNTS" From afa at codespeak.net Wed Apr 21 00:07:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 00:07:08 +0200 (CEST) Subject: [pypy-svn] r73922 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420220708.BABBC282BD4@codespeak.net> Author: afa Date: Wed Apr 21 00:07:07 2010 New Revision: 73922 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: For tests, allow automatic conversion from wrapped objects to pointers like PyUnicodeObject. This fixes tests for PyUnicode_AsWideChar Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 21 00:07:07 2010 @@ -175,16 +175,17 @@ to_decref = [] for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] - if ARG is PyObject and not is_wrapped: + if is_PyObject(ARG) and not is_wrapped: # build a reference if input_arg is None: arg = lltype.nullptr(PyObject.TO) elif isinstance(input_arg, W_Root): - arg = make_ref(space, input_arg) - to_decref.append(arg) + ref = make_ref(space, input_arg) + to_decref.append(ref) + arg = rffi.cast(ARG, ref) else: arg = input_arg - elif ARG is PyObject and is_wrapped: + elif is_PyObject(ARG) and is_wrapped: # convert to a wrapped object if input_arg is None: arg = input_arg @@ -322,6 +323,11 @@ PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) PyVarObject = lltype.Ptr(PyVarObjectStruct) +def is_PyObject(TYPE): + if not isinstance(TYPE, lltype.Ptr): + return False + return hasattr(TYPE.TO, 'c_ob_refcnt') and hasattr(TYPE.TO, 'c_ob_type') + # a pointer to PyObject PyObjectP = rffi.CArrayPtr(PyObject) From afa at codespeak.net Wed Apr 21 00:07:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 00:07:55 +0200 (CEST) Subject: [pypy-svn] r73923 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420220755.B03E3282BD4@codespeak.net> Author: afa Date: Wed Apr 21 00:07:54 2010 New Revision: 73923 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/state.py Log: Fix translation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/state.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/state.py Wed Apr 21 00:07:54 2010 @@ -59,7 +59,7 @@ op_err = OperationError(exc_type, exc_value) raise op_err if always: - raise OperationError(space.w_SystemError, space.wrap( + raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) def print_refcounts(self): From afa at codespeak.net Wed Apr 21 00:25:27 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 00:25:27 +0200 (CEST) Subject: [pypy-svn] r73924 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420222527.8B61A282BD4@codespeak.net> Author: afa Date: Wed Apr 21 00:25:25 2010 New Revision: 73924 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: More translation fixes Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 21 00:25:25 2010 @@ -323,6 +323,7 @@ PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) PyVarObject = lltype.Ptr(PyVarObjectStruct) + at specialize.memo() def is_PyObject(TYPE): if not isinstance(TYPE, lltype.Ptr): return False Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Wed Apr 21 00:25:25 2010 @@ -67,7 +67,8 @@ check_num_args(space, w_args, 2) args_w = space.fixedview(w_args) index = space.int_w(space.index(args_w[0])) - if generic_cpy_call(space, func_target, w_self, index, args_w[1]) == -1: + res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) + if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception() def wrap_sq_delitem(space, w_self, w_args, func): @@ -76,7 +77,8 @@ args_w = space.fixedview(w_args) index = space.int_w(space.index(args_w[0])) null = lltype.nullptr(PyObject.TO) - if generic_cpy_call(space, func_target, w_self, index, null) == -1: + res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) + if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) def wrap_ssizessizeargfunc(space, w_self, w_args, func): From afa at codespeak.net Wed Apr 21 01:40:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 01:40:18 +0200 (CEST) Subject: [pypy-svn] r73925 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420234018.69298282BD4@codespeak.net> Author: afa Date: Wed Apr 21 01:40:16 2010 New Revision: 73925 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: PyLong_FromString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Wed Apr 21 01:40:16 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, rffi -from pypy.module.cpyext.api import cpython_api, PyObject, build_type_checkers, ADDR +from pypy.module.cpyext.api import (cpython_api, PyObject, build_type_checkers, + CONST_STRING, ADDR) from pypy.objspace.std.longobject import W_LongObject from pypy.interpreter.error import OperationError @@ -47,6 +48,24 @@ raised.""" return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + at cpython_api([CONST_STRING, rffi.CCHARP, rffi.INT_real], PyObject) +def PyLong_FromString(space, str, pend, base): + """Return a new PyLongObject based on the string value in str, which is + interpreted according to the radix in base. If pend is non-NULL, + *pend will point to the first character in str which follows the + representation of the number. If base is 0, the radix will be determined + based on the leading characters of str: if str starts with '0x' or + '0X', radix 16 will be used; if str starts with '0', radix 8 will be + used; otherwise radix 10 will be used. If base is not 0, it must be + between 2 and 36, inclusive. Leading spaces are ignored. If there are + no digits, ValueError will be raised.""" + s = rffi.charp2str(str) + w_str = space.wrap(s) + w_base = space.wrap(base) + if pend: + pend[0] = rffi.ptradd(str, len(s)) + return space.call_function(space.w_long, w_str, w_base) + @cpython_api([rffi.VOIDP], PyObject) def PyLong_FromVoidPtr(space, p): """Create a Python integer or long integer from the pointer p. The pointer value Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 01:40:16 2010 @@ -3435,19 +3435,6 @@ NULL on failure.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {char**}, rffi.INT_real], PyObject) -def PyLong_FromString(space, str, pend, base): - """Return a new PyLongObject based on the string value in str, which is - interpreted according to the radix in base. If pend is non-NULL, - *pend will point to the first character in str which follows the - representation of the number. If base is 0, the radix will be determined - based on the leading characters of str: if str starts with '0x' or - '0X', radix 16 will be used; if str starts with '0', radix 8 will be - used; otherwise radix 10 will be used. If base is not 0, it must be - between 2 and 36, inclusive. Leading spaces are ignored. If there are - no digits, ValueError will be raised.""" - raise NotImplementedError - @cpython_api([{Py_UNICODE*}, Py_ssize_t, rffi.INT_real], PyObject) def PyLong_FromUnicode(space, u, length, base): """Convert a sequence of Unicode digits to a Python long integer value. The first Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Wed Apr 21 01:40:16 2010 @@ -64,3 +64,12 @@ """)]) assert module.from_longlong() == -1 assert module.from_unsignedlonglong() == (1<<64) - 1 + + def test_fromstring(self): + module = self.import_extension('foo', [ + ("from_string", "METH_NOARGS", + """ + return PyLong_FromString("0x1234", NULL, 0); + """), + ]) + assert module.from_string() == 0x1234 From afa at codespeak.net Wed Apr 21 01:46:02 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 01:46:02 +0200 (CEST) Subject: [pypy-svn] r73926 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100420234602.06C27282BD4@codespeak.net> Author: afa Date: Wed Apr 21 01:46:01 2010 New Revision: 73926 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: PyString_Format Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Wed Apr 21 01:46:01 2010 @@ -155,3 +155,10 @@ appended to string. This version decrements the reference count of newpart.""" PyString_Concat(space, ref, newpart) Py_DecRef(space, newpart) + + at cpython_api([PyObject, PyObject], PyObject) +def PyString_Format(space, w_format, w_args): + """Return a new string object from format and args. Analogous to format % + args. The args argument must be a tuple.""" + return space.mod(w_format, w_args) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 01:46:01 2010 @@ -5144,27 +5144,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObjectP, PyObject], lltype.Void) -def PyString_Concat(space, string, newpart): - """Create a new string object in *string containing the contents of newpart - appended to string; the caller will own the new reference. The reference to - the old value of string will be stolen. If the new string cannot be created, - the old reference to string will still be discarded and the value of - *string will be set to NULL; the appropriate exception will be set.""" - raise NotImplementedError - - at cpython_api([PyObjectP, PyObject], lltype.Void) -def PyString_ConcatAndDel(space, string, newpart): - """Create a new string object in *string containing the contents of newpart - appended to string. This version decrements the reference count of newpart.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyString_Format(space, format, args): - """Return a new string object from format and args. Analogous to format % - args. The args argument must be a tuple.""" - raise NotImplementedError - @cpython_api([PyObjectP], lltype.Void) def PyString_InternInPlace(space, string): """Intern the argument *string in place. The argument must be the address of a Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Wed Apr 21 01:46:01 2010 @@ -203,3 +203,7 @@ api.PyString_ConcatAndDel(ptr, ref2) # should not crash assert ref2.c_ob_refcnt == 0 lltype.free(ptr, flavor='raw') + + def test_format(self, space, api): + assert "1 2" == space.unwrap( + api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2)))) From afa at codespeak.net Wed Apr 21 01:53:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 01:53:32 +0200 (CEST) Subject: [pypy-svn] r73927 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100420235332.47FAD282BD4@codespeak.net> Author: afa Date: Wed Apr 21 01:53:30 2010 New Revision: 73927 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Log: translation fix Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Wed Apr 21 01:53:30 2010 @@ -48,7 +48,7 @@ raised.""" return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) - at cpython_api([CONST_STRING, rffi.CCHARP, rffi.INT_real], PyObject) + at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject) def PyLong_FromString(space, str, pend, base): """Return a new PyLongObject based on the string value in str, which is interpreted according to the radix in base. If pend is non-NULL, From afa at codespeak.net Wed Apr 21 02:13:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 02:13:03 +0200 (CEST) Subject: [pypy-svn] r73928 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421001303.21B32282BD4@codespeak.net> Author: afa Date: Wed Apr 21 02:13:00 2010 New Revision: 73928 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: PyUnicode_Decode Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 02:13:00 2010 @@ -5592,18 +5592,6 @@ throughout the interpreter whenever coercion to Unicode is needed.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) -def PyUnicode_Decode(space, s, size, encoding, errors): - """Create a Unicode object by decoding size bytes of the encoded string s. - encoding and errors have the same meaning as the parameters of the same name - in the unicode() built-in function. The codec to be used is looked up - using the Python codec registry. Return NULL if an exception was raised by - the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) def PyUnicode_Encode(space, s, size, encoding, errors): """Encode the Py_UNICODE buffer of the given size and return a Python Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Wed Apr 21 02:13:00 2010 @@ -86,3 +86,11 @@ def test_TOUPPER(self, space, api): assert api.Py_UNICODE_TOUPPER(u'?') == u'?' assert api.Py_UNICODE_TOUPPER(u'?') == u'?' + + def test_decode(self, space, api): + b_text = rffi.str2charp('caf\x82xx') + b_encoding = rffi.str2charp('cp437') + assert space.unwrap( + api.PyUnicode_Decode(b_text, 4, b_encoding, None)) == u'caf\xe9' + lltype.free(b_text, flavor='raw') + lltype.free(b_encoding, flavor='raw') Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 21 02:13:00 2010 @@ -230,3 +230,18 @@ @cpython_api([PyObject, CONST_STRING], PyObject) def _PyUnicode_AsDefaultEncodedString(space, w_unicode, errors): return PyUnicode_AsEncodedString(space, w_unicode, lltype.nullptr(rffi.CCHARP.TO), errors) + + at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, CONST_STRING], PyObject) +def PyUnicode_Decode(space, s, size, encoding, errors): + """Create a Unicode object by decoding size bytes of the encoded string s. + encoding and errors have the same meaning as the parameters of the same name + in the unicode() built-in function. The codec to be used is looked up + using the Python codec registry. Return NULL if an exception was raised by + the codec.""" + w_str = space.wrap(rffi.charpsize2str(s, size)) + w_encoding = space.wrap(rffi.charp2str(encoding)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + else: + w_errors = space.w_None + return space.call_method(w_str, 'decode', w_encoding, w_errors) From afa at codespeak.net Wed Apr 21 02:17:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 02:17:53 +0200 (CEST) Subject: [pypy-svn] r73929 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100421001753.45957282BD4@codespeak.net> Author: afa Date: Wed Apr 21 02:17:51 2010 New Revision: 73929 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: PySequence_Length and PySequence_Size, identical to PyObject_Size. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 21 02:17:51 2010 @@ -96,6 +96,14 @@ def PyObject_Size(space, w_obj): return space.int_w(space.len(w_obj)) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PySequence_Size(space, w_obj): + return space.int_w(space.len(w_obj)) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PySequence_Length(space, w_obj): + return space.int_w(space.len(w_obj)) + @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallable_Check(space, w_obj): """Determine if the object o is callable. Return 1 if the object is callable Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 02:17:51 2010 @@ -4627,20 +4627,6 @@ This function always succeeds.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PySequence_Size(space, o): - """ - - - - Returns the number of objects in sequence o on success, and -1 on failure. - For objects that do not provide sequence protocol, this is equivalent to the - Python expression len(o). - - These functions returned an int type. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PySequence_Concat(space, o1, o2): """Return the concatenation of o1 and o2 on success, and NULL on failure. From afa at codespeak.net Wed Apr 21 02:27:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 02:27:00 +0200 (CEST) Subject: [pypy-svn] r73930 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421002700.50929282BD4@codespeak.net> Author: afa Date: Wed Apr 21 02:26:58 2010 New Revision: 73930 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Log: PySequence_Tuple Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 21 02:26:58 2010 @@ -96,14 +96,6 @@ def PyObject_Size(space, w_obj): return space.int_w(space.len(w_obj)) - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PySequence_Size(space, w_obj): - return space.int_w(space.len(w_obj)) - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PySequence_Length(space, w_obj): - return space.int_w(space.len(w_obj)) - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallable_Check(space, w_obj): """Determine if the object o is callable. Return 1 if the object is callable Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Wed Apr 21 02:26:58 2010 @@ -21,6 +21,10 @@ Python expression len(o).""" return space.int_w(space.len(w_obj)) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PySequence_Length(space, w_obj): + return space.int_w(space.len(w_obj)) + @cpython_api([PyObject, CONST_STRING], PyObject) def PySequence_Fast(space, w_obj, m): @@ -76,3 +80,10 @@ the Python expression o[i].""" return space.getitem(w_obj, space.wrap(i)) + at cpython_api([PyObject], PyObject) +def PySequence_Tuple(space, w_obj): + """Return a tuple object with the same contents as the arbitrary sequence o or + NULL on failure. If o is a tuple, a new reference will be returned, + otherwise a tuple will be constructed with the appropriate contents. This is + equivalent to the Python expression tuple(o).""" + return space.call_function(space.w_tuple, w_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 02:26:58 2010 @@ -4737,18 +4737,6 @@ returned list is guaranteed to be new.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PySequence_Tuple(space, o): - """ - - - - Return a tuple object with the same contents as the arbitrary sequence o or - NULL on failure. If o is a tuple, a new reference will be returned, - otherwise a tuple will be constructed with the appropriate contents. This is - equivalent to the Python expression tuple(o).""" - raise NotImplementedError - @cpython_api([PyObject], PyObjectP) def PySequence_Fast_ITEMS(space, o): """Return the underlying array of PyObject pointers. Assumes that o was returned Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Wed Apr 21 02:26:58 2010 @@ -18,6 +18,10 @@ assert space.type(w_seq) is space.w_tuple assert space.int_w(space.len(w_seq)) == 4 + w_seq = api.PySequence_Tuple(w_set) + assert space.type(w_seq) is space.w_tuple + assert sorted(space.unwrap(w_seq)) == [1, 2, 3, 4] + def test_exception(self, space, api): message = rffi.str2charp("message") assert not api.PySequence_Fast(space.wrap(3), message) From afa at codespeak.net Wed Apr 21 02:48:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 02:48:55 +0200 (CEST) Subject: [pypy-svn] r73931 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421004855.2FFCD282BD4@codespeak.net> Author: afa Date: Wed Apr 21 02:48:53 2010 New Revision: 73931 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_AsFileDescriptor Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 21 02:48:53 2010 @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject +from pypy.interpreter.error import OperationError import pypy.module.__builtin__.operation as operation @@ -222,3 +223,31 @@ this function uses the generic algorithm described above.""" from pypy.module.__builtin__.abstractinst import abstract_issubclass_w return abstract_issubclass_w(space, w_derived, w_cls) + + at cpython_api([PyObject], rffi.INT_real, error=-1) +def PyObject_AsFileDescriptor(space, w_obj): + """Derives a file descriptor from a Python object. If the object is an + integer or long integer, its value is returned. If not, the object's + fileno() method is called if it exists; the method must return an integer or + long integer, which is returned as the file descriptor value. Returns -1 on + failure.""" + try: + fd = space.int_w(w_obj) + except OperationError: + try: + w_meth = space.getattr(w_obj, space.wrap('fileno')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be an int, or have a fileno() method.")) + else: + w_fd = space.call_function(w_meth) + fd = space.int_w(w_fd) + + if fd < 0: + raise OperationError( + space.w_ValueError, space.wrap( + "file descriptor cannot be a negative integer")) + + return rffi.cast(rffi.INT_real, fd) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 02:48:53 2010 @@ -4496,14 +4496,6 @@ equivalent of the Python statement del o[key].""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyObject_AsFileDescriptor(space, o): - """Derives a file descriptor from a Python object. If the object is an integer or - long integer, its value is returned. If not, the object's fileno() method - is called if it exists; the method must return an integer or long integer, which - is returned as the file descriptor value. Returns -1 on failure.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyObject_Dir(space, o): """This is equivalent to the Python expression dir(o), returning a (possibly Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 21 02:48:53 2010 @@ -124,3 +124,17 @@ space.w_type, space.newtuple([space.w_int, space.w_type])) == 1 assert api.PyObject_IsSubclass(space.wrap(1), space.w_type) == -1 api.PyErr_Clear() + + def test_fileno(self, space, api): + assert api.PyObject_AsFileDescriptor(space.wrap(1)) == 1 + assert api.PyObject_AsFileDescriptor(space.wrap(-20)) == -1 + assert api.PyErr_Occurred() is space.w_ValueError + api.PyErr_Clear() + + w_File = space.appexec([], """(): + class File: + def fileno(self): + return 42 + return File""") + w_f = space.call_function(w_File) + assert api.PyObject_AsFileDescriptor(w_f) == 42 From afa at codespeak.net Wed Apr 21 02:54:32 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 02:54:32 +0200 (CEST) Subject: [pypy-svn] r73932 - in pypy/branch/cpython-extension/pypy/module/cpyext: . src Message-ID: <20100421005432.8E740282BD4@codespeak.net> Author: afa Date: Wed Apr 21 02:54:31 2010 New Revision: 73932 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/src/pyerrors.c Log: PyErr_NewException compiles nowadays Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 21 02:54:31 2010 @@ -247,7 +247,7 @@ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', - 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', + 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/pyerrors.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/pyerrors.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/pyerrors.c Wed Apr 21 02:54:31 2010 @@ -20,9 +20,6 @@ return NULL; } -#if 0 -depends on unavailable functions - PyObject * PyErr_NewException(char *name, PyObject *base, PyObject *dict) { @@ -72,6 +69,3 @@ Py_XDECREF(modulename); return result; } - - -#endif From afa at codespeak.net Wed Apr 21 08:38:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 08:38:03 +0200 (CEST) Subject: [pypy-svn] r73933 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100421063803.BBD8D282BD4@codespeak.net> Author: afa Date: Wed Apr 21 08:38:02 2010 New Revision: 73933 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Log: Translation fix Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Wed Apr 21 08:38:02 2010 @@ -61,7 +61,7 @@ no digits, ValueError will be raised.""" s = rffi.charp2str(str) w_str = space.wrap(s) - w_base = space.wrap(base) + w_base = space.wrap(rffi.cast(lltype.Signed, base)) if pend: pend[0] = rffi.ptradd(str, len(s)) return space.call_function(space.w_long, w_str, w_base) From afa at codespeak.net Wed Apr 21 08:53:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 08:53:33 +0200 (CEST) Subject: [pypy-svn] r73934 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100421065333.22A81282BD4@codespeak.net> Author: afa Date: Wed Apr 21 08:53:31 2010 New Revision: 73934 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Log: Py_Module_AddStringConstant is already defined, just expose it Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 21 08:53:31 2010 @@ -246,7 +246,8 @@ SYMBOLS_C = [ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', - 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', + 'PyString_FromFormat', 'PyString_FromFormatV', + 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/modsupport.h Wed Apr 21 08:53:31 2010 @@ -28,6 +28,9 @@ PYTHON_API_VERSION) int PyModule_AddObject(PyObject *m, const char *name, PyObject *o); +int PyModule_AddIntConstant(PyObject *m, const char *name, long value); +int PyModule_AddStringConstant(PyObject *m, const char *name, const char *value); + PyObject * Py_BuildValue(const char *, ...); PyObject * _Py_BuildValue_SizeT(const char *, ...); From afa at codespeak.net Wed Apr 21 10:13:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 10:13:51 +0200 (CEST) Subject: [pypy-svn] r73935 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100421081351.9B0F8282BD4@codespeak.net> Author: afa Date: Wed Apr 21 10:13:50 2010 New Revision: 73935 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Add PyType_Check Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 21 10:13:50 2010 @@ -16,7 +16,7 @@ PyVarObjectFields, Py_ssize_t, Py_TPFLAGS_READYING, generic_cpy_call, \ Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE, ADDR, \ Py_TPFLAGS_HAVE_CLASS, METH_VARARGS, METH_KEYWORDS, \ - CANNOT_FAIL, PyBufferProcs + CANNOT_FAIL, PyBufferProcs, build_type_checkers from pypy.module.cpyext.pyobject import PyObject, make_ref, create_ref, from_ref from pypy.module.cpyext.pyobject import get_typedescr, make_typedescr, track_reference from pypy.interpreter.module import Module @@ -39,6 +39,7 @@ WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False +PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type") class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset, pto): From arigo at codespeak.net Wed Apr 21 14:24:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Apr 2010 14:24:52 +0200 (CEST) Subject: [pypy-svn] r73936 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100421122452.B52FF282BD4@codespeak.net> Author: arigo Date: Wed Apr 21 14:24:50 2010 New Revision: 73936 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Generate _copy operations instead of just a _rename whenever possible. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Wed Apr 21 14:24:50 2010 @@ -146,8 +146,7 @@ raise CannotOptimize # variable is not produced in cur block def insert_renamings(self, link): - renamings_from = [] - renamings_to = [] + renamings = {} lst = [(v, self.getcolor(link.target.inputargs[i])) for i, v in enumerate(link.args)] lst.sort(key=lambda(v, w): w.index) @@ -156,10 +155,20 @@ v = self.getcolor(v) if v == w: continue - renamings_from.append(v) - renamings_to.append(w) - if renamings_from: - self.emitline('int_rename', renamings_from, renamings_to) + frm, to = renamings.setdefault(w.kind, ([], [])) + frm.append(v) + to.append(w) + for kind in KINDS: + if kind in renamings: + frm, to = renamings[kind] + # If there is no cycle among the renamings, produce a series + # of %s_copy. Otherwise, just one %s_rename. + result = reorder_renaming_list(frm, to) + if result is not None: + for v, w in result: + self.emitline('%s_copy' % kind, v, w) + else: + self.emitline('%s_rename' % kind, frm, to) def emitline(self, *line): self.assembler.insns.append(line) @@ -182,3 +191,21 @@ except KeyError: r = self.registers[kind, col] = Register(kind, col) return r + +# ____________________________________________________________ + +def reorder_renaming_list(frm, to): + result = [] + pending_indices = range(len(to)) + while pending_indices: + not_read = dict.fromkeys([frm[i] for i in pending_indices]) + still_pending_indices = [] + for i in pending_indices: + if to[i] not in not_read: + result.append((frm[i], to[i])) + else: + still_pending_indices.append(i) + if len(pending_indices) == len(still_pending_indices): + return None # no progress -- there is a cycle + pending_indices = still_pending_indices + return result Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed Apr 21 14:24:50 2010 @@ -1,6 +1,6 @@ import py from pypy.jit.codewriter import support -from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list from pypy.jit.codewriter.format import format_assembler @@ -15,6 +15,18 @@ self.num_colors += 1 return self.seen[v] +def test_reorder_renaming_list(): + result = reorder_renaming_list([], []) + assert result == [] + result = reorder_renaming_list([1, 2, 3], [4, 5, 6]) + assert result == [(1, 4), (2, 5), (3, 6)] + result = reorder_renaming_list([4, 5, 1, 2], [1, 2, 3, 4]) + assert result == [(1, 3), (4, 1), (2, 4), (5, 2)] + result = reorder_renaming_list([1, 2], [2, 1]) + assert result == None + result = reorder_renaming_list([4, 3, 1, 2, 6], [1, 2, 3, 4, 5]) + assert result == None + class TestFlatten: @@ -46,14 +58,17 @@ a -= 1 return b self.encoding_test(f, [5, 6], """ - int_rename [%i0, %i1], [%i2, %i3] + int_copy %i0, %i2 + int_copy %i1, %i3 L1: int_gt %i2, $0, %i4 goto_if_not L2, %i4 - int_rename [%i2, %i3], [%i5, %i6] + int_copy %i2, %i5 + int_copy %i3, %i6 int_add %i6, %i5, %i7 int_sub %i5, $1, %i8 - int_rename [%i8, %i7], [%i2, %i3] + int_copy %i8, %i2 + int_copy %i7, %i3 goto L1 L2: int_return %i3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed Apr 21 14:24:50 2010 @@ -85,7 +85,8 @@ L1: int_gt %i0, $0, %i0 goto_if_not L2, %i0 - int_rename [%i1, $2], [%i0, %i1] + int_copy %i1, %i0 + int_copy $2, %i1 goto L1 L2: int_return %i1 @@ -117,7 +118,7 @@ L1: int_gt %i0, $0, %i3 goto_if_not L2, %i3 - int_rename [%i2], [%i1] + int_copy %i2, %i1 goto L1 L2: int_return %i1 From arigo at codespeak.net Wed Apr 21 14:38:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Apr 2010 14:38:37 +0200 (CEST) Subject: [pypy-svn] r73937 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100421123837.94652282BD4@codespeak.net> Author: arigo Date: Wed Apr 21 14:38:36 2010 New Revision: 73937 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Log: Tweaks. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Wed Apr 21 14:38:36 2010 @@ -61,6 +61,7 @@ break else: raise AssertionError("bad opcode") + dispatch_loop._dont_inline_ = True self.dispatch_loop = dispatch_loop def _get_method(self, name, argcodes): @@ -128,12 +129,15 @@ # XXX must be specialized def copy_constants(self, registers, constants): - i = len(constants) - while i > 0: - j = 256 - i + """Copy jitcode.constants[0] to registers[255], + jitcode.constants[1] to registers[254], + jitcode.constants[2] to registers[253], etc.""" + i = len(constants) - 1 + while i >= 0: + j = 255 - i assert j >= 0 - i -= 1 registers[j] = constants[i] + i -= 1 # ---------- @@ -150,6 +154,18 @@ self.result_i = a raise LeaveFrame + @arguments("i", returns="i") + def opimpl_int_copy(self, a): + return a + + @arguments("r", returns="r") + def opimpl_ref_copy(self, a): + return a + + @arguments("f", returns="f") + def opimpl_float_copy(self, a): + return a + @arguments("L", "i", "i", "pc", returns="L") def opimpl_goto_if_not_int_gt(self, target, a, b, pc): if a > b: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py Wed Apr 21 14:38:36 2010 @@ -53,7 +53,7 @@ 'int_sub/ici': 2, 'goto/L': 3, 'int_return/i': 4}) - blackholeinterp.setarg_i(0x16, 6) - blackholeinterp.setarg_i(0x17, 100) + blackholeinterp.setarg_i(0x16, 6) # %i0 + blackholeinterp.setarg_i(0x17, 100) # %i1 blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 100+6+5+4+3 From afa at codespeak.net Wed Apr 21 15:03:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 15:03:34 +0200 (CEST) Subject: [pypy-svn] r73938 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421130334.33C35282BD4@codespeak.net> Author: afa Date: Wed Apr 21 15:03:32 2010 New Revision: 73938 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Oops, ensure that tp_basic_size is correctly set for custom types like PyStringObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Wed Apr 21 15:03:32 2010 @@ -28,6 +28,8 @@ if(PyString_Size(s) == 11) { result = 1; } + if(s->ob_type->tp_basicsize != 16) + result = 0; Py_DECREF(s); return PyBool_FromLong(result); """), Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 21 15:03:32 2010 @@ -485,10 +485,10 @@ pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) PyPyType_Ready(space, pto, w_type) + pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: - pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize - else: - pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) + if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: + pto.c_tp_basicsize = pto.c_tp_base.c_tp_basicsize # will be filled later on with the correct value # may not be 0 From arigo at codespeak.net Wed Apr 21 15:19:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Apr 2010 15:19:43 +0200 (CEST) Subject: [pypy-svn] r73939 - in pypy/branch/blackhole-improvement/pypy: jit/codewriter/test jit/metainterp jit/metainterp/test rlib rlib/test Message-ID: <20100421131943.6385B282BD4@codespeak.net> Author: arigo Date: Wed Apr 21 15:19:40 2010 New Revision: 73939 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_support.py - copied, changed from r73937, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_support.py Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_support.py Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py pypy/branch/blackhole-improvement/pypy/rlib/jit.py pypy/branch/blackhole-improvement/pypy/rlib/test/test_jit.py Log: * fix imports * kill the deprecated "leave" jit hook Copied: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_support.py (from r73937, pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_support.py) ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_support.py Wed Apr 21 15:19:40 2010 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype from pypy.objspace.flow.model import Variable, Constant, SpaceOperation -from pypy.jit.metainterp.support import decode_builtin_call +from pypy.jit.codewriter.support import decode_builtin_call def newconst(x): return Constant(x, lltype.typeOf(x)) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py Wed Apr 21 15:19:40 2010 @@ -7,7 +7,8 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import _we_are_jitted from pypy.jit.metainterp.history import Const, getkind, dict_equal_consts -from pypy.jit.metainterp import heaptracker, support, history +from pypy.jit.metainterp import heaptracker, history +from pypy.jit.codewriter import support from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -76,16 +77,13 @@ self.raise_analyzer = None self.jitdriver = None - def find_all_graphs(self, portal_graph, leave_graph, - policy, supports_floats): + def find_all_graphs(self, portal_graph, policy, supports_floats): from pypy.translator.simplify import get_graph def is_candidate(graph): return policy.look_inside_graph(graph) policy.set_supports_floats(supports_floats) todo = [portal_graph] - if leave_graph is not None: - todo.append(leave_graph) self.candidate_graphs = seen = set(todo) def callers(): @@ -178,15 +176,12 @@ return graph in self.candidate_graphs - def generate_bytecode(self, metainterp_sd, portal_graph, leave_graph, + def generate_bytecode(self, metainterp_sd, portal_graph, portal_runner_ptr): self._start(metainterp_sd, portal_runner_ptr) - leave_code = None - if leave_graph: - leave_code = self.make_one_bytecode((leave_graph, None), False) portal_code = self.make_portal_bytecode(portal_graph) - self.metainterp_sd.info_from_codewriter(portal_code, leave_code, + self.metainterp_sd.info_from_codewriter(portal_code, self.class_sizes, self.list_of_addr2name, portal_runner_ptr) @@ -1290,8 +1285,6 @@ calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], args, op.result) - self.emit('recursion_leave_prep') - self.emit_varargs(non_void_args) self.emit('recursive_call') self.emit(self.get_position(calldescr)) self.emit_varargs([op.args[0]] + non_void_args) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/policy.py Wed Apr 21 15:19:40 2010 @@ -1,5 +1,5 @@ from pypy.translator.simplify import get_funcobj -from pypy.jit.metainterp import support, history +from pypy.jit.metainterp import history from pypy.rpython.lltypesystem import lltype, rclass from pypy.tool.udir import udir Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed Apr 21 15:19:40 2010 @@ -648,19 +648,6 @@ def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - @arguments("varargs") - def opimpl_recursion_leave_prep(self, varargs): - warmrunnerstate = self.metainterp.staticdata.state - if warmrunnerstate.inlining: - num_green_args = self.metainterp.staticdata.num_green_args - greenkey = varargs[:num_green_args] - if warmrunnerstate.can_inline_callable(greenkey): - return False - leave_code = self.metainterp.staticdata.leave_code - if leave_code is None: - return False - return self.perform_call(leave_code, varargs) - @arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): warmrunnerstate = self.metainterp.staticdata.state @@ -1128,7 +1115,6 @@ self.jit_starting_line = 'JIT starting (%s)' % backendmodule self.portal_code = None - self.leave_code = None self._class_sizes = None self._addr2name_keys = [] self._addr2name_values = [] @@ -1141,10 +1127,9 @@ def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes, + def info_from_codewriter(self, portal_code, class_sizes, list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code - self.leave_code = leave_code self._class_sizes = class_sizes self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed Apr 21 15:19:40 2010 @@ -4,8 +4,9 @@ from pypy.rlib.jit import OPTIMIZER_FULL, OPTIMIZER_SIMPLE, loop_invariant from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner -from pypy.jit.metainterp import support, codewriter, pyjitpl, history +from pypy.jit.metainterp import codewriter, pyjitpl, history from pypy.jit.metainterp.policy import JitPolicy, StopAtXPolicy +from pypy.jit.codewriter import support from pypy import conftest from pypy.rlib.rarithmetic import ovfcheck from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper @@ -93,7 +94,7 @@ portal_graph = rtyper.annotator.translator.graphs[0] cw = codewriter.CodeWriter(rtyper) - graphs = cw.find_all_graphs(portal_graph, None, JitPolicy(), + graphs = cw.find_all_graphs(portal_graph, JitPolicy(), self.CPUClass.supports_floats) cw._start(metainterp.staticdata, None) portal_graph.func._jit_unroll_safe_ = True Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_codewriter.py Wed Apr 21 15:19:40 2010 @@ -1,6 +1,7 @@ import py from pypy.rlib import jit -from pypy.jit.metainterp import support, typesystem +from pypy.jit.codewriter import support +from pypy.jit.metainterp import typesystem from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.history import ConstInt from pypy.jit.metainterp.codewriter import CodeWriter, BytecodeMaker @@ -25,7 +26,7 @@ rtyper = support.annotate(i, [7]) cw = CodeWriter(rtyper) jitpolicy = JitPolicy() - res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], None, + res = cw.find_all_graphs(rtyper.annotator.translator.graphs[0], jitpolicy, True) translator = rtyper.annotator.translator @@ -41,13 +42,13 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, g]) cw = CodeWriter(rtyper) - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=False) funcs = [graph.func for graph in res] assert funcs == [f] @@ -75,7 +76,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([f, h]) @@ -96,7 +97,7 @@ cw = CodeWriter(rtyper) jitpolicy = JitPolicy() translator = rtyper.annotator.translator - res = cw.find_all_graphs(translator.graphs[0], None, jitpolicy, + res = cw.find_all_graphs(translator.graphs[0], jitpolicy, supports_floats=True) funcs = set([graph.func for graph in res]) assert funcs == set([g, h]) @@ -110,7 +111,7 @@ jitpolicy = JitPolicy() translator = rtyper.annotator.translator # does not explode - cw.find_all_graphs(translator.graphs[0], None, jitpolicy, True) + cw.find_all_graphs(translator.graphs[0], jitpolicy, True) class SomeLabel(object): def __eq__(self, other): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_pyjitpl.py Wed Apr 21 15:19:40 2010 @@ -121,7 +121,7 @@ def __init__(self): pass metainterp_sd = FakeMetaInterpSd() - metainterp_sd.info_from_codewriter(None, None, None, + metainterp_sd.info_from_codewriter(None, None, [(123, "a"), (456, "b")], None) assert metainterp_sd.get_name_from_address(123) == 'a' Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_recursive.py Wed Apr 21 15:19:40 2010 @@ -494,77 +494,6 @@ res = self.meta_interp(main, [1], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) self.check_loops(call_may_force=0, call=0) - def test_leave_jit_hook(self): - from pypy.rpython.annlowlevel import hlstr - def p(code, pc): - code = hlstr(code) - return "%s %d %s" % (code, pc, code[pc]) - def c(code, pc): - return "L" not in hlstr(code) - - def really_leave(frame): - frame.hookcalled = True - - def leave(code, pc, frame): - really_leave(frame) - - class ExpectedHook(Exception): - pass - class UnexpectedHook(Exception): - pass - - myjitdriver = JitDriver(greens=['code', 'pc'], reds=['self'], - get_printable_location=p, can_inline=c, - leave=leave) - class Frame(object): - hookcalled = True - - def __init__(self, n): - self.n = n - self.hookcalled = False - def f(self, code): - pc = 0 - while pc < len(code): - - myjitdriver.jit_merge_point(self=self, code=code, pc=pc) - op = code[pc] - if op == "-": - self.n -= 1 - elif op == "c": - frame = Frame(self.n) - self.n = frame.f("---i---") - if we_are_jitted(): - if frame.hookcalled: - raise UnexpectedHook - elif op == "C": - frame = Frame(self.n) - self.n = frame.f("cL") - if we_are_jitted(): - if not frame.hookcalled: - raise ExpectedHook - elif op == "i": - if self.n % 5 == 1: - return self.n - elif op == "l": - if self.n > 0: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - elif op == "L": - if self.n > 50: - myjitdriver.can_enter_jit(self=self, code=code, pc=0) - pc = 0 - continue - else: - assert 0 - pc += 1 - return self.n - def main(n): - frame = Frame(n) - return frame.f("C-l") - res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) - assert res == main(100) - def test_trace_from_start(self): def p(code, pc): code = hlstr(code) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_warmspot.py Wed Apr 21 15:19:40 2010 @@ -288,7 +288,7 @@ class TestWarmspotDirect(object): def setup_class(cls): from pypy.jit.metainterp.typesystem import llhelper - from pypy.jit.metainterp.support import annotate + from pypy.jit.codewriter.support import annotate from pypy.jit.metainterp.warmspot import WarmRunnerDesc from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython.lltypesystem import lltype, llmemory Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Wed Apr 21 15:19:40 2010 @@ -17,11 +17,12 @@ from pypy.translator.unsimplify import call_final_function from pypy.jit.metainterp import codewriter -from pypy.jit.metainterp import support, history, pyjitpl, gc +from pypy.jit.metainterp import history, pyjitpl, gc from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler +from pypy.jit.codewriter import support from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE # ____________________________________________________________ @@ -150,10 +151,8 @@ policy = JitPolicy() self.set_translator(translator) self.find_portal() - self.make_leave_jit_graph() self.codewriter = codewriter.CodeWriter(self.rtyper) graphs = self.codewriter.find_all_graphs(self.portal_graph, - self.leave_graph, policy, CPUClass.supports_floats) policy.dump_unsafe_loops() @@ -177,7 +176,6 @@ self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, - self.leave_graph, self.portal_runner_ptr ) self.rewrite_can_enter_jit() @@ -369,18 +367,6 @@ maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True self.maybe_enter_from_start_fn = maybe_enter_from_start - - - def make_leave_jit_graph(self): - self.leave_graph = None - if self.jitdriver.leave: - args_s = self.portal_args_s - from pypy.annotation import model as annmodel - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annmodel.s_None - self.leave_graph = annhelper.getgraph(self.jitdriver.leave, - args_s, s_result) - annhelper.finish() def make_driverhook_graphs(self): from pypy.rlib.jit import BaseJitCell Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmstate.py Wed Apr 21 15:19:40 2010 @@ -11,7 +11,8 @@ from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.jit.metainterp import support, history +from pypy.jit.metainterp import history +from pypy.jit.codewriter import support # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/rlib/jit.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rlib/jit.py (original) +++ pypy/branch/blackhole-improvement/pypy/rlib/jit.py Wed Apr 21 15:19:40 2010 @@ -224,8 +224,7 @@ def __init__(self, greens=None, reds=None, virtualizables=None, get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, - confirm_enter_jit=None, - leave=None): # XXX 'leave' is deprecated + confirm_enter_jit=None): if greens is not None: self.greens = greens if reds is not None: @@ -243,7 +242,6 @@ self.get_printable_location = get_printable_location self.can_inline = can_inline self.confirm_enter_jit = confirm_enter_jit - self.leave = leave def _freeze_(self): return True @@ -358,7 +356,6 @@ **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) - self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: Modified: pypy/branch/blackhole-improvement/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/blackhole-improvement/pypy/rlib/test/test_jit.py Wed Apr 21 15:19:40 2010 @@ -61,12 +61,10 @@ def can_inline(m): pass def get_printable_location(m): pass - def leave(m, n): pass myjitdriver = JitDriver(greens=['m'], reds=['n'], can_inline=can_inline, - get_printable_location=get_printable_location, - leave=leave) + get_printable_location=get_printable_location) def fn(n): m = 42.5 while n > 0: @@ -83,9 +81,6 @@ return [v.concretetype for v in graph.getargs()] raise Exception, 'function %r has not been annotated' % func - leave_args = getargs(leave) - assert leave_args == [lltype.Float, lltype.Signed] - can_inline_args = getargs(can_inline) get_printable_location_args = getargs(get_printable_location) assert can_inline_args == get_printable_location_args == [lltype.Float] From arigo at codespeak.net Wed Apr 21 16:12:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Apr 2010 16:12:22 +0200 (CEST) Subject: [pypy-svn] r73940 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100421141222.05806282BD4@codespeak.net> Author: arigo Date: Wed Apr 21 16:12:21 2010 New Revision: 73940 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Add jitter.py, for now only to put the optimizer for 'goto_if_not_xxx' in it. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed Apr 21 16:12:21 2010 @@ -2,6 +2,7 @@ from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler +from pypy.jit.codewriter.jitter import transform_graph class CodeWriter(object): @@ -16,6 +17,7 @@ return self.transform_graph_to_jitcode(graph) def transform_graph_to_jitcode(self, graph): + transform_graph(graph) regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Wed Apr 21 16:12:21 2010 @@ -1,5 +1,6 @@ from pypy.objspace.flow.model import Variable from pypy.jit.metainterp.history import getkind +from pypy.rpython.lltypesystem import lltype class SSARepr(object): @@ -112,39 +113,27 @@ assert link.exitcase is None self.make_link(link) else: - assert len(block.exits) == 2 + assert len(block.exits) == 2, "XXX" linkfalse, linktrue = block.exits if linkfalse.llexitcase == True: linkfalse, linktrue = linktrue, linkfalse + if isinstance(block.exitswitch, tuple): + # special case produced by jitter.optimize_goto_if_not() + opname = 'goto_if_not_' + block.exitswitch[0] + opargs = block.exitswitch[1:] + else: + assert block.exitswitch.concretetype == lltype.Bool + opname = 'goto_if_not' + opargs = [block.exitswitch] # - self.emitline('goto_if_not', TLabel(linkfalse), - self.getcolor(block.exitswitch)) + self.emitline(opname, TLabel(linkfalse), + *self.flatten_list(opargs)) # true path: self.make_link(linktrue) # false path: self.emitline(Label(linkfalse)) self.make_link(linkfalse) - def optimize_goto_if_not(self, block): - xxxxxxx - if not self.optimize: - raise CannotOptimize - v = block.exitswitch - for link in block.exits: - if v in link.args: - raise CannotOptimize # variable escapes to next block - for op in block.operations[::-1]: - if v in op.args: - raise CannotOptimize # variable is also used in cur block - if v is op.result: - if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', - 'int_gt', 'int_ge'): - raise CannotOptimize # not a supported operation - killop = (op.opname,) + tuple(op.args) + (v,) - self.assembler.insns.remove(killop) - return 'goto_if_not_' + op.opname, op.args - raise CannotOptimize # variable is not produced in cur block - def insert_renamings(self, link): renamings = {} lst = [(v, self.getcolor(link.target.inputargs[i])) @@ -173,12 +162,16 @@ def emitline(self, *line): self.assembler.insns.append(line) - def serialize_op(self, op): + def flatten_list(self, list): args = [] - for v in op.args: + for v in list: if isinstance(v, Variable): v = self.getcolor(v) args.append(v) + return args + + def serialize_op(self, op): + args = self.flatten_list(op.args) if op.result is not None: args.append(self.getcolor(op.result)) self.emitline(op.opname, *args) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 21 16:12:21 2010 @@ -0,0 +1,34 @@ +from pypy.rpython.lltypesystem import lltype + + +def transform_graph(graph): + """Transform a control flow graph to make it suitable for + being flattened in a JitCode. + """ + for block in graph.iterblocks(): + optimize_goto_if_not(block) + + +def optimize_goto_if_not(block): + """Replace code like 'v = int_gt(x,y); exitswitch = v' + with just 'exitswitch = ('int_gt',x,y)'.""" + if len(block.exits) != 2: + return False + v = block.exitswitch + if v.concretetype != lltype.Bool: + return False + for link in block.exits: + if v in link.args: + return False # variable escapes to next block + for op in block.operations[::-1]: + if v in op.args: + return False # variable is also used in cur block + if v is op.result: + if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', + 'int_gt', 'int_ge'): + return False # not a supported operation + # ok! optimize this case + block.operations.remove(op) + block.exitswitch = (op.opname,) + tuple(op.args) + return True + return False Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Wed Apr 21 16:12:21 2010 @@ -1,4 +1,5 @@ from pypy.jit.codewriter.codewriter import CodeWriter +from pypy.jit.codewriter.blackhole import BlackholeInterpreter def test_loop(): @@ -9,15 +10,28 @@ return b cw = CodeWriter() jitcode = cw.transform_func_to_jitcode(f, [5, 6]) - assert jitcode.code == ("\x00\x00\x00\x02" - "\x01\x13\x00\x02" - "\x02\x01\x00\x01" - "\x03\x00\x01\x00" - "\x04\x00\x00" - "\x05\x01") - assert cw.assembler.insns == {'int_gt/ici': 0, - 'goto_if_not/Li': 1, - 'int_add/iii': 2, - 'int_sub/ici': 3, - 'goto/L': 4, - 'int_return/i': 5} + assert jitcode.code == ("\x00\x10\x00\x00\x00" + "\x01\x01\x00\x01" + "\x02\x00\x01\x00" + "\x03\x00\x00" + "\x04\x01") + assert cw.assembler.insns == {'goto_if_not_int_gt/Lic': 0, + 'int_add/iii': 1, + 'int_sub/ici': 2, + 'goto/L': 3, + 'int_return/i': 4} + +def test_integration(): + def f(a, b): + while a > 2: + b += a + a -= 1 + return b + cw = CodeWriter() + jitcode = cw.transform_func_to_jitcode(f, [5, 6]) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns(cw.assembler.insns) + blackholeinterp.setarg_i(0, 6) + blackholeinterp.setarg_i(1, 100) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 100+6+5+4+3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed Apr 21 16:12:21 2010 @@ -2,6 +2,7 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.jitter import transform_graph class FakeRegAlloc: @@ -34,8 +35,10 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def encoding_test(self, func, args, expected, optimize=True): + def encoding_test(self, func, args, expected, optimize=False): graphs = self.make_graphs(func, args) + if optimize: + transform_graph(graphs[0]) ssarepr = flatten_graph(graphs[0], {'int': FakeRegAlloc(), 'ref': FakeRegAlloc(), 'float': FakeRegAlloc()}) @@ -74,6 +77,28 @@ int_return %i3 """) + def test_loop_opt(self): + def f(a, b): + while a > 0: + b += a + a -= 1 + return b + self.encoding_test(f, [5, 6], """ + int_copy %i0, %i2 + int_copy %i1, %i3 + L1: + goto_if_not_int_gt L2, %i2, $0 + int_copy %i2, %i4 + int_copy %i3, %i5 + int_add %i5, %i4, %i6 + int_sub %i4, $1, %i7 + int_copy %i7, %i2 + int_copy %i6, %i3 + goto L1 + L2: + int_return %i3 + """, optimize=True) + def test_float(self): def f(i, f): return (i*5) + (f*0.25) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 21 16:12:21 2010 @@ -0,0 +1,49 @@ +from pypy.objspace.flow.model import FunctionGraph, Block, Link +from pypy.objspace.flow.model import SpaceOperation, Variable, Constant +from pypy.jit.codewriter import jitter +from pypy.rpython.lltypesystem import lltype + +class FakeLink: + args = [] + +def test_optimize_goto_if_not(): + v1 = Variable() + v2 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + sp1 = SpaceOperation('foobar', [], None) + sp2 = SpaceOperation('foobaz', [], None) + block = Block([v1, v2]) + block.operations = [sp1, SpaceOperation('int_gt', [v1, v2], v3), sp2] + block.exitswitch = v3 + block.exits = exits = [FakeLink(), FakeLink()] + res = jitter.optimize_goto_if_not(block) + assert res == True + assert block.operations == [sp1, sp2] + assert block.exitswitch == ('int_gt', v1, v2) + assert block.exits == exits + +def test_optimize_goto_if_not__incoming(): + v1 = Variable(); v1.concretetype = lltype.Bool + block = Block([v1]) + block.exitswitch = v1 + block.exits = [FakeLink(), FakeLink()] + assert not jitter.optimize_goto_if_not(block) + +def test_optimize_goto_if_not__exit(): + v1 = Variable() + v2 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + block = Block([v1, v2]) + block.operations = [SpaceOperation('int_gt', [v1, v2], v3)] + block.exitswitch = v3 + block.exits = [FakeLink(), FakeLink()] + block.exits[1].args = [v3] + assert not jitter.optimize_goto_if_not(block) + +def test_optimize_goto_if_not__unknownop(): + v3 = Variable(); v3.concretetype = lltype.Bool + block = Block([]) + block.operations = [SpaceOperation('foobar', [], v3)] + block.exitswitch = v3 + block.exits = [FakeLink(), FakeLink()] + assert not jitter.optimize_goto_if_not(block) From afa at codespeak.net Wed Apr 21 16:27:26 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 16:27:26 +0200 (CEST) Subject: [pypy-svn] r73941 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100421142726.53958282BDA@codespeak.net> Author: afa Date: Wed Apr 21 16:27:24 2010 New Revision: 73941 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: Oops, I did not run all tests obviously Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Wed Apr 21 16:27:24 2010 @@ -77,7 +77,7 @@ args_w = space.fixedview(w_args) index = space.int_w(space.index(args_w[0])) null = lltype.nullptr(PyObject.TO) - res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) + res = generic_cpy_call(space, func_target, w_self, index, null) if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) From afa at codespeak.net Wed Apr 21 16:28:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 16:28:22 +0200 (CEST) Subject: [pypy-svn] r73942 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421142822.10CF6282BDA@codespeak.net> Author: afa Date: Wed Apr 21 16:28:20 2010 New Revision: 73942 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Log: implement METH_NOARGS and METH_OLDARGS calling conventions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Wed Apr 21 16:28:20 2010 @@ -8,8 +8,10 @@ from pypy.interpreter.function import BuiltinFunction, Method from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref -from pypy.module.cpyext.api import generic_cpy_call, cpython_api, PyObject,\ - cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING +from pypy.module.cpyext.api import ( + generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, + METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, + METH_VARARGS) from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated @@ -38,13 +40,18 @@ if w_self is None: w_self = self.w_self flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) + flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) if space.is_true(w_kw) and not flags & METH_KEYWORDS: - raise OperationError(space.w_TypeError, - space.wrap(rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) - # XXX support METH_NOARGS + raise OperationError(space.w_TypeError, space.wrap( + rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) if flags & METH_KEYWORDS: func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) return generic_cpy_call(space, func, w_self, w_args, w_kw) + elif flags & METH_NOARGS: + if len(w_args.wrappeditems) == 0: + return generic_cpy_call(space, self.ml.c_ml_meth, w_self, None) + raise OperationError(space.w_TypeError, space.wrap( + rffi.charp2str(self.ml.c_ml_name) + "() takes no arguments")) elif flags & METH_O: assert isinstance(w_args, W_TupleObject) if len(w_args.wrappeditems) != 1: @@ -54,8 +61,17 @@ len(w_args.wrappeditems)))) w_arg = w_args.wrappeditems[0] return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) - else: + elif flags & METH_VARARGS: return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_args) + else: # METH_OLDARGS, the really old style + size = len(w_args.wrappeditems) + if size == 1: + w_arg = w_args.wrappeditems[0] + elif size == 0: + w_arg = None + else: + w_arg = w_args + return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) class W_PyCMethodObject(W_PyCFunctionObject): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Wed Apr 21 16:28:20 2010 @@ -2,15 +2,47 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class AppTestMethodObject(AppTestCpythonExtensionBase): - def test_METH_O(self): + def test_call_METH(self): mod = self.import_extension('foo', [ - ('getarg', 'METH_O', + ('getarg_O', 'METH_O', ''' Py_INCREF(args); return args; ''' ), + ('getarg_NO', 'METH_NOARGS', + ''' + if(args) { + Py_INCREF(args); + return args; + } + else { + Py_INCREF(Py_None); + return Py_None; + } + ''' + ), + ('getarg_OLD', 'METH_OLDARGS', + ''' + if(args) { + Py_INCREF(args); + return args; + } + else { + Py_INCREF(Py_None); + return Py_None; + } + ''' + ), ]) - assert mod.getarg(1) == 1 - raises(TypeError, mod.getarg) - raises(TypeError, mod.getarg, 1, 1) + assert mod.getarg_O(1) == 1 + raises(TypeError, mod.getarg_O) + raises(TypeError, mod.getarg_O, 1, 1) + + assert mod.getarg_NO() is None + raises(TypeError, mod.getarg_NO, 1) + raises(TypeError, mod.getarg_NO, 1, 1) + + assert mod.getarg_OLD(1) == 1 + assert mod.getarg_OLD() is None + assert mod.getarg_OLD(1, 2) == (1, 2) From afa at codespeak.net Wed Apr 21 16:51:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 16:51:05 +0200 (CEST) Subject: [pypy-svn] r73943 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421145105.2DD8F282BDA@codespeak.net> Author: afa Date: Wed Apr 21 16:51:03 2010 New Revision: 73943 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_HasAttrString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 21 16:51:03 2010 @@ -81,8 +81,20 @@ @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_HasAttr(space, w_obj, w_name): - w_res = operation.hasattr(space, w_obj, w_name) - return space.is_true(w_res) + try: + w_res = operation.hasattr(space, w_obj, w_name) + return space.is_true(w_res) + except OperationError: + return 0 + + at cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=CANNOT_FAIL) +def PyObject_HasAttrString(space, w_obj, name_ptr): + try: + name = rffi.charp2str(name_ptr) + w_res = operation.hasattr(space, w_obj, space.wrap(name)) + return space.is_true(w_res) + except OperationError: + return 0 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_SetAttr(space, w_obj, w_name, w_value): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 16:51:03 2010 @@ -4257,13 +4257,6 @@ instead of the repr().""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) -def PyObject_HasAttrString(space, o, attr_name): - """Returns 1 if o has the attribute attr_name, and 0 otherwise. This - is equivalent to the Python expression hasattr(o, attr_name). This function - always succeeds.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PyObject_GetAttr(space, o, attr_name): """Retrieve an attribute named attr_name from object o. Returns the attribute Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 21 16:51:03 2010 @@ -33,6 +33,11 @@ assert hasattr_(space.w_int, '__eq__') assert not hasattr_(space.w_int, 'nonexistingattr') + buf = rffi.str2charp('__len__') + assert api.PyObject_HasAttrString(space.w_str, buf) + assert not api.PyObject_HasAttrString(space.w_int, buf) + rffi.free_charp(buf) + def test_SetAttr(self, space, api): class X: pass From afa at codespeak.net Wed Apr 21 16:56:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 16:56:59 +0200 (CEST) Subject: [pypy-svn] r73944 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421145659.9F2E3282BDA@codespeak.net> Author: afa Date: Wed Apr 21 16:56:57 2010 New Revision: 73944 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: replace lltype.free with rffi.free_charp, for memory allocated with rffi.str2charp. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Wed Apr 21 16:56:57 2010 @@ -44,7 +44,7 @@ try: return generic_cpy_call(space, func_target, w_self, name_ptr) finally: - lltype.free(name_ptr, flavor="raw") + rffi.free_charp(name_ptr) def wrap_call(space, w_self, w_args, func, w_kwds): func_target = rffi.cast(ternaryfunc, func) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 21 16:56:57 2010 @@ -56,8 +56,8 @@ assert not api.PyObject_GetAttrString(space.wrap(""), charp2) assert api.PyErr_Occurred() is space.w_AttributeError api.PyErr_Clear() - lltype.free(charp1, flavor="raw") - lltype.free(charp2, flavor="raw") + rffi.free_charp(charp1) + rffi.free_charp(charp2) def test_getitem(self, space, api): w_t = space.wrap((1, 2, 3, 4, 5)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Wed Apr 21 16:56:57 2010 @@ -54,7 +54,7 @@ assert rffi.wcharp2unicode(buf) == 'longw' api.PyUnicode_AsWideChar(space.wrap(u'a'), buf, 5) assert rffi.wcharp2unicode(buf) == 'a' - lltype.free(buf, flavor='raw') + rffi.free_wcharp(buf) def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, @@ -92,5 +92,5 @@ b_encoding = rffi.str2charp('cp437') assert space.unwrap( api.PyUnicode_Decode(b_text, 4, b_encoding, None)) == u'caf\xe9' - lltype.free(b_text, flavor='raw') - lltype.free(b_encoding, flavor='raw') + rffi.free_charp(b_text) + rffi.free_charp(b_encoding) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 21 16:56:57 2010 @@ -442,7 +442,7 @@ if obj_pto.c_tp_as_buffer: lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') Py_DecRef(space, base_pyo) - lltype.free(obj_pto.c_tp_name, flavor="raw") + rffi.free_charp(obj_pto.c_tp_name) obj_pto_voidp = rffi.cast(rffi.VOIDP_real, obj_pto) generic_cpy_call(space, type_pto.c_tp_free, obj_pto_voidp) pto = rffi.cast(PyObject, type_pto) From fijal at codespeak.net Wed Apr 21 17:10:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Apr 2010 17:10:13 +0200 (CEST) Subject: [pypy-svn] r73945 - pypy/trunk/pypy/rlib Message-ID: <20100421151013.6FBC3282BDA@codespeak.net> Author: fijal Date: Wed Apr 21 17:10:12 2010 New Revision: 73945 Modified: pypy/trunk/pypy/rlib/rlocale.py Log: I think all those should be sandboxsafe Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Wed Apr 21 17:10:12 2010 @@ -145,7 +145,8 @@ def external(name, args, result, calling_conv='c'): return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, - calling_conv=calling_conv) + calling_conv=calling_conv, + sandboxsafe=True) _setlocale = external('setlocale', [rffi.INT, rffi.CCHARP], rffi.CCHARP) From afa at codespeak.net Wed Apr 21 19:42:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 19:42:38 +0200 (CEST) Subject: [pypy-svn] r73948 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421174238.AD89A282B9E@codespeak.net> Author: afa Date: Wed Apr 21 19:42:36 2010 New Revision: 73948 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Log: Add PyNumber_* abstract operations Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Wed Apr 21 19:42:36 2010 @@ -2,6 +2,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t from pypy.module.cpyext.pyobject import PyObject from pypy.rpython.lltypesystem import rffi, lltype +from pypy.tool.sourcetools import func_with_new_name @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyIndex_Check(space, w_obj): @@ -25,3 +26,66 @@ integer or PY_SSIZE_T_MAX for a positive integer. """ return space.int_w(w_obj) #XXX: this is wrong + +def func_rename(newname): + return lambda func: func_with_new_name(func, newname) + +def make_numbermethod(name, spacemeth): + @cpython_api([PyObject, PyObject], PyObject) + @func_rename('PyNumber_%s' % (name,)) + def PyNumber_Method(space, w_o1, w_o2): + meth = getattr(space, spacemeth) + return meth(w_o1, w_o2) + +def make_unary_numbermethod(name, spacemeth): + @cpython_api([PyObject], PyObject) + @func_rename('PyNumber_%s' % (name,)) + def PyNumber_Method(space, w_o1): + meth = getattr(space, spacemeth) + return meth(w_o1) + +def make_inplace_numbermethod(name, spacemeth): + spacemeth = 'inplace_' + spacemeth.rstrip('_') + @cpython_api([PyObject, PyObject], PyObject) + @func_rename('PyNumber_Inplace%s' % (name,)) + def PyNumber_Method(space, w_o1, w_o2): + meth = getattr(space, spacemeth) + return meth(w_o1, w_o2) + +for name, spacemeth in [ + ('Add', 'add'), + ('Subtract', 'sub'), + ('Multiply', 'mul'), + ('Divide', 'div'), + ('FloorDivide', 'floordiv'), + ('TrueDivide', 'truediv'), + ('Remainder', 'mod'), + ('Lshift', 'lshift'), + ('Rshift', 'rshift'), + ('And', 'and_'), + ('Xor', 'xor'), + ('Or', 'or_'), + ('Divmod', 'divmod'), + ]: + make_numbermethod(name, spacemeth) + if name != 'Divmod': + make_inplace_numbermethod(name, spacemeth) + +for name, spacemeth in [ + ('Negative', 'neg'), + ('Positive', 'pos'), + ('Absolute', 'abs'), + ('Invert', 'invert')]: + make_unary_numbermethod(name, spacemeth) + + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyNumber_Power(space, w_o1, w_o2, w_o3): + return space.pow(w_o1, w_o2, w_o3) + + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyNumber_InplacePower(space, w_o1, w_o2, w_o3): + if not space.is_w(w_o3, space.w_None): + raise OperationError(space.w_ValueError, space.wrap( + "PyNumber_InplacePower with non-None modulus is not supported")) + return space.inplace_pow(w_o1, w_o2) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 21 19:42:36 2010 @@ -3905,222 +3905,6 @@ the equivalent of the Python expression o1 - o2.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Multiply(space, o1, o2): - """Returns the result of multiplying o1 and o2, or NULL on failure. This is - the equivalent of the Python expression o1 * o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Divide(space, o1, o2): - """Returns the result of dividing o1 by o2, or NULL on failure. This is the - equivalent of the Python expression o1 / o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_FloorDivide(space, o1, o2): - """Return the floor of o1 divided by o2, or NULL on failure. This is - equivalent to the "classic" division of integers. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_TrueDivide(space, o1, o2): - """Return a reasonable approximation for the mathematical value of o1 divided by - o2, or NULL on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Remainder(space, o1, o2): - """Returns the remainder of dividing o1 by o2, or NULL on failure. This is - the equivalent of the Python expression o1 % o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Divmod(space, o1, o2): - """ - - - - See the built-in function divmod(). Returns NULL on failure. This is - the equivalent of the Python expression divmod(o1, o2).""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyNumber_Power(space, o1, o2, o3): - """ - - - - See the built-in function pow(). Returns NULL on failure. This is the - equivalent of the Python expression pow(o1, o2, o3), where o3 is optional. - If o3 is to be ignored, pass Py_None in its place (passing NULL for - o3 would cause an illegal memory access).""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyNumber_Negative(space, o): - """Returns the negation of o on success, or NULL on failure. This is the - equivalent of the Python expression -o.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyNumber_Positive(space, o): - """Returns o on success, or NULL on failure. This is the equivalent of the - Python expression +o.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyNumber_Absolute(space, o): - """ - - - - Returns the absolute value of o, or NULL on failure. This is the equivalent - of the Python expression abs(o).""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyNumber_Invert(space, o): - """Returns the bitwise negation of o on success, or NULL on failure. This is - the equivalent of the Python expression ~o.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Lshift(space, o1, o2): - """Returns the result of left shifting o1 by o2 on success, or NULL on - failure. This is the equivalent of the Python expression o1 << o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Rshift(space, o1, o2): - """Returns the result of right shifting o1 by o2 on success, or NULL on - failure. This is the equivalent of the Python expression o1 >> o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_And(space, o1, o2): - """Returns the "bitwise and" of o1 and o2 on success and NULL on failure. - This is the equivalent of the Python expression o1 & o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Xor(space, o1, o2): - """Returns the "bitwise exclusive or" of o1 by o2 on success, or NULL on - failure. This is the equivalent of the Python expression o1 ^ o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Or(space, o1, o2): - """Returns the "bitwise or" of o1 and o2 on success, or NULL on failure. - This is the equivalent of the Python expression o1 | o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceAdd(space, o1, o2): - """Returns the result of adding o1 and o2, or NULL on failure. The operation - is done in-place when o1 supports it. This is the equivalent of the Python - statement o1 += o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceSubtract(space, o1, o2): - """Returns the result of subtracting o2 from o1, or NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 -= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceMultiply(space, o1, o2): - """Returns the result of multiplying o1 and o2, or NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 *= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceDivide(space, o1, o2): - """Returns the result of dividing o1 by o2, or NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 /= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceFloorDivide(space, o1, o2): - """Returns the mathematical floor of dividing o1 by o2, or NULL on failure. - The operation is done in-place when o1 supports it. This is the equivalent - of the Python statement o1 //= o2. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceTrueDivide(space, o1, o2): - """Return a reasonable approximation for the mathematical value of o1 divided by - o2, or NULL on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. The operation is done in-place when o1 supports it. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceRemainder(space, o1, o2): - """Returns the remainder of dividing o1 by o2, or NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 %= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyNumber_InPlacePower(space, o1, o2, o3): - """ - - - - See the built-in function pow(). Returns NULL on failure. The operation - is done in-place when o1 supports it. This is the equivalent of the Python - statement o1 **= o2 when o3 is Py_None, or an in-place variant of - pow(o1, o2, o3) otherwise. If o3 is to be ignored, pass Py_None - in its place (passing NULL for o3 would cause an illegal memory access).""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceLshift(space, o1, o2): - """Returns the result of left shifting o1 by o2 on success, or NULL on - failure. The operation is done in-place when o1 supports it. This is the - equivalent of the Python statement o1 <<= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceRshift(space, o1, o2): - """Returns the result of right shifting o1 by o2 on success, or NULL on - failure. The operation is done in-place when o1 supports it. This is the - equivalent of the Python statement o1 >>= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceAnd(space, o1, o2): - """Returns the "bitwise and" of o1 and o2 on success and NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 &= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceXor(space, o1, o2): - """Returns the "bitwise exclusive or" of o1 by o2 on success, or NULL on - failure. The operation is done in-place when o1 supports it. This is the - equivalent of the Python statement o1 ^= o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_InPlaceOr(space, o1, o2): - """Returns the "bitwise or" of o1 and o2 on success, or NULL on failure. The - operation is done in-place when o1 supports it. This is the equivalent of - the Python statement o1 |= o2.""" - raise NotImplementedError - @cpython_api([PyObjectP, PyObjectP], rffi.INT_real) def PyNumber_Coerce(space, p1, p2): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Wed Apr 21 19:42:36 2010 @@ -7,3 +7,29 @@ def test_index(self, space, api): assert api.PyIndex_Check(space.wrap(12)) assert not api.PyIndex_Check(space.wrap('12')) + + def test_numbermethods(self, space, api): + assert "ab" == space.unwrap( + api.PyNumber_Add(space.wrap("a"), space.wrap("b"))) + assert "aaa" == space.unwrap( + api.PyNumber_Multiply(space.wrap("a"), space.wrap(3))) + + w_l = space.newlist([1, 2, 3]) + w_l2 = api.PyNumber_Multiply(w_l, space.wrap(3)) + assert api.PyObject_Size(w_l2) == 9 + assert api.PyObject_Size(w_l) == 3 + + w_l3 = api.PyNumber_InplaceMultiply(w_l, space.wrap(3)) + assert api.PyObject_Size(w_l) == 9 + assert w_l3 is w_l + + # unary function + assert 9 == space.unwrap(api.PyNumber_Absolute(space.wrap(-9))) + + # power + assert 9 == space.unwrap( + api.PyNumber_Power(space.wrap(3), space.wrap(2), space.w_None)) + assert 4 == space.unwrap( + api.PyNumber_Power(space.wrap(3), space.wrap(2), space.wrap(5))) + assert 9 == space.unwrap( + api.PyNumber_InplacePower(space.wrap(3), space.wrap(2), space.w_None)) From afa at codespeak.net Wed Apr 21 19:50:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 19:50:01 +0200 (CEST) Subject: [pypy-svn] r73949 - in pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils: . command Message-ID: <20100421175001.9865A282B9E@codespeak.net> Author: afa Date: Wed Apr 21 19:49:59 2010 New Revision: 73949 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/command/build_ext.py pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/spawn.py pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Log: Hack the windows part of distutils until it can compile modules. Replace msvccompiler.py with msvc9compiler.py stolen from cpython2.6, it supports ms compiler versions >= 8.0 --Cette ligne, et les suivantes ci-dessous, seront ignor?es-- M distutils/spawn.py M distutils/msvccompiler.py M distutils/sysconfig_pypy.py M distutils/command/build_ext.py Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/command/build_ext.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/command/build_ext.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/command/build_ext.py Wed Apr 21 19:49:59 2010 @@ -167,7 +167,7 @@ # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + self.library_dirs.append(os.path.join(sys.pypy_prefix, 'pypy', '_interfaces')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: @@ -175,8 +175,8 @@ # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) - self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) + #self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + #self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) # OS/2 (EMX) doesn't support Debug vs Release builds, but has the # import libraries in its "Config" subdirectory @@ -645,24 +645,12 @@ shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. + # The python library is always needed on Windows. if sys.platform == "win32": - from distutils.msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries + pythonlib = 'libpypy-c.exe' + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] elif sys.platform == "os2emx": # EMX/GCC requires the python library explicitly, and I # believe VACPP does as well (though not confirmed) - AIM Apr01 Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/msvccompiler.py Wed Apr 21 19:49:59 2010 @@ -1,154 +1,162 @@ -"""distutils.msvccompiler +"""distutils.msvc9compiler Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. """ # Written by Perry Stoll # hacked by Robin Becker and Thomas Heller to do a better job of # finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes -# This module should be kept compatible with Python 2.1. - -__revision__ = "$Id: msvccompiler.py 54645 2007-04-01 18:29:47Z neal.norwitz $" +__revision__ = "$Id: msvc9compiler.py 78713 2010-03-06 02:17:28Z tarek.ziade $" -import sys, os, string -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options +import os +import subprocess +import sys +import re + +from distutils.errors import (DistutilsExecError, DistutilsPlatformError, + CompileError, LibError, LinkError) +from distutils.ccompiler import CCompiler, gen_lib_options from distutils import log +from distutils.util import get_platform -_can_read_reg = 0 -try: - import _winreg - - _can_read_reg = 1 - hkey_mod = _winreg - - RegOpenKeyEx = _winreg.OpenKeyEx - RegEnumKey = _winreg.EnumKey - RegEnumValue = _winreg.EnumValue - RegError = _winreg.error +import _winreg -except ImportError: - try: - import win32api - import win32con - _can_read_reg = 1 - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules _winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) +RegOpenKeyEx = _winreg.OpenKeyEx +RegEnumKey = _winreg.EnumKey +RegEnumValue = _winreg.EnumValue +RegError = _winreg.error + +HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) + +VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" +VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" +WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" +NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targetting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', + 'win-ia64' : 'ia64', +} -def read_keys(base, key): - """Return list of registry keys.""" +class Reg: + """Helper class to read values from the registry + """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while 1: + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" try: - k = RegEnumKey(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - L.append(k) - i = i + 1 - return L + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) -def read_values(base, key): - """Return dict of registry keys and values. + def read_values(cls, base, key): + """Return dict of registry keys and values. - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while 1: + All names are converted to lowercase. + """ try: - name, value, type = RegEnumValue(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i = i + 1 - return d - -def convert_mbcs(s): - enc = getattr(s, "encode", None) - if enc is not None: - try: - s = enc("mbcs") - except UnicodeError: - pass - return s + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) class MacroExpander: def __init__(self, version): self.macros = {} + self.vsbase = VS_BASE % version self.load_macros(version) def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break + self.macros["$(%s)" % macro] = Reg.get_value(path, key) def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError, exc: # - raise DistutilsPlatformError, \ - ("""Python was built with Visual Studio 2003; + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, +Visual Studio 2008 was not found on this system. If you have Cygwin installed, you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] def sub(self, s): for k, v in self.macros.items(): - s = string.replace(s, k, v) + s = s.replace(k, v) return s def get_build_version(): @@ -157,35 +165,7 @@ For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - - prefix = "MSC v." - i = string.find(sys.version, prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = string.find(sys.version, prefix) - if i == -1: - return "Intel" - j = string.find(sys.version, ")", i) - return sys.version[i+len(prefix):j] + return 8.0 def normalize_and_reduce_paths(paths): """Return a list of normalized paths with duplicates removed. @@ -201,8 +181,104 @@ reduced_paths.append(np) return reduced_paths +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + + # trying Express edition + if productdir is None: + vsbase = VSEXPRESS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + log.debug("Unable to find productdir in registry") + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = set(("include", "lib", "libpath", "path")) + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) -class MSVCCompiler (CCompiler) : +class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -232,27 +308,29 @@ static_lib_format = shared_lib_format = '%s%s' exe_extension = '.exe' - def __init__ (self, verbose=0, dry_run=0, force=0): + def __init__(self, verbose=0, dry_run=0, force=0): CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name self.initialized = False - def initialize(self): - self.__paths = [] - if os.environ.has_key("DISTUTILS_USE_SDK") and os.environ.has_key("MSSdk") and self.find_exe("cl.exe"): + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + ok_plats = 'win32', 'win-amd64', 'win-ia64' + if plat_name not in ok_plats: + raise DistutilsPlatformError("--plat-name must be one of %s" % + (ok_plats,)) + + if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): # Assume that the SDK set up everything alright; don't try to be # smarter self.cc = "cl.exe" @@ -261,36 +339,54 @@ self.rc = "rc.exe" self.mc = "mc.exe" else: - self.__paths = self.get_msvc_paths("path") + # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; + # to cross compile, you use 'x86_amd64'. + # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross + # compile use 'x86' (ie, it runs the x86 compiler directly) + # No idea how itanium handles this, if at all. + if plat_name == get_platform() or plat_name == 'win32': + # native build or cross-compile to win32 + plat_spec = PLAT_TO_VCVARS[plat_name] + else: + # cross compile from win32 -> some 64bit + plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ + PLAT_TO_VCVARS[plat_name] + + vc_env = query_vcvarsall(VERSION, plat_spec) + + # take care to only use strings in the environment. + self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep) + os.environ['lib'] = vc_env['lib'].encode('mbcs') + os.environ['include'] = vc_env['include'].encode('mbcs') - if len (self.__paths) == 0: - raise DistutilsPlatformError, \ - ("Python was built with %s, " + if len(self.__paths) == 0: + raise DistutilsPlatformError("Python was built with %s, " "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." % self.__product) + "version of the compiler, but it isn't installed." + % self.__product) self.cc = self.find_exe("cl.exe") self.linker = self.find_exe("link.exe") self.lib = self.find_exe("lib.exe") self.rc = self.find_exe("rc.exe") # resource compiler self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') + #self.set_path_env_var('lib') + #self.set_path_env_var('include') # extend the MSVC path with the current path try: - for p in string.split(os.environ['path'], ';'): + for p in os.environ['path'].split(';'): self.__paths.append(p) except KeyError: pass self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = string.join(self.__paths, ';') + os.environ['path'] = ";".join(self.__paths) self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , + if self.__arch == "x86": + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'] else: # Win64 @@ -304,20 +400,16 @@ self.ldflags_shared_debug = [ '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] self.ldflags_static = [ '/nologo'] self.initialized = True # -- Worker methods ------------------------------------------------ - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): + def object_filenames(self, + source_filenames, + strip_dir=0, + output_dir=''): # Copied from ccompiler.py, extended to return .res as 'object'-file # for .rc input file if output_dir is None: output_dir = '' @@ -344,17 +436,16 @@ base + self.obj_extension)) return obj_names - # object_filenames () - def compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - if not self.initialized: self.initialize() - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info compile_opts = extra_preargs or [] compile_opts.append ('/c') @@ -383,13 +474,12 @@ input_opt = src output_opt = "/fo" + obj try: - self.spawn ([self.rc] + pp_opts + - [output_opt] + [input_opt]) + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. # * '-h dir' specifies the directory for the # generated include file @@ -401,99 +491,95 @@ # we use the source-directory for the include file and # the build directory for the RC file and message # resources. This works at least for win32all. - - h_dir = os.path.dirname (src) - rc_dir = os.path.dirname (obj) + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) try: # first compile .MC to .RC and .H file - self.spawn ([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) base, _ = os.path.splitext (os.path.basename (src)) rc_file = os.path.join (rc_dir, base + '.rc') # then compile .RC to .RES file - self.spawn ([self.rc] + - ["/fo" + obj] + [rc_file]) + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue else: # how to handle this file? - raise CompileError ( - "Don't know how to compile %s to %s" % \ - (src, obj)) + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) output_opt = "/Fo" + obj try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) return objects - # compile () - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=0, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) - if self._need_link (objects, output_filename): + if self._need_link(objects, output_filename): lib_args = objects + ['/OUT:' + output_filename] if debug: - pass # XXX what goes here? + pass # XXX what goes here? try: - self.spawn ([self.lib] + lib_args) + self.spawn([self.lib] + lib_args) except DistutilsExecError, msg: - raise LibError, msg - + raise LibError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # create_static_lib () - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) + def link(self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + (libraries, library_dirs, runtime_library_dirs) = fixed_args if runtime_library_dirs: self.warn ("I don't know what to do with 'runtime_library_dirs': " + str (runtime_library_dirs)) - lib_opts = gen_lib_options (self, - library_dirs, runtime_library_dirs, - libraries) + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): + output_filename = os.path.join(output_dir, output_filename) + if self._need_link(objects, output_filename): if target_desc == CCompiler.EXECUTABLE: if debug: ldflags = self.ldflags_shared_debug[1:] @@ -517,47 +603,99 @@ # needed! Make sure they are generated in the temporary build # directory. Since they have different names for debug and release # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) if export_symbols is not None: (dll_name, dll_ext) = os.path.splitext( os.path.basename(output_filename)) implib_file = os.path.join( - os.path.dirname(objects[0]), + build_temp, self.library_filename(dll_name)) ld_args.append ('/IMPLIB:' + implib_file) + # Embedded manifests are recommended - see MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can embed it later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + if extra_preargs: ld_args[:0] = extra_preargs if extra_postargs: ld_args.extend(extra_postargs) - self.mkpath (os.path.dirname (output_filename)) + self.mkpath(os.path.dirname(output_filename)) try: - self.spawn ([self.linker] + ld_args) + self.spawn([self.linker] + ld_args) except DistutilsExecError, msg: - raise LinkError, msg + raise LinkError(msg) + # embed the manifest + # XXX - this is somewhat fragile - if mt.exe fails, distutils + # will still consider the DLL up-to-date, but it will not have a + # manifest. Maybe we should link to a temp file? OTOH, that + # implies a build environment error that shouldn't go undetected. + if target_desc == CCompiler.EXECUTABLE: + mfid = 1 + else: + mfid = 2 + self._remove_visual_c_ref(temp_manifest) + out_arg = '-outputresource:%s;%s' % (output_filename, mfid) + try: + self.spawn(['mt.exe', '-nologo', '-manifest', + temp_manifest, out_arg]) + except DistutilsExecError, msg: + raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # link () - + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = "\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + finally: + manifest_f.close() + except IOError: + pass # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. - def library_dir_option (self, dir): + def library_dir_option(self, dir): return "/LIBPATH:" + dir - def runtime_library_dir_option (self, dir): - raise DistutilsPlatformError, \ - "don't know how to set runtime library search path for MSVC++" + def runtime_library_dir_option(self, dir): + raise DistutilsPlatformError( + "don't know how to set runtime library search path for MSVC++") - def library_option (self, lib): - return self.library_filename (lib) + def library_option(self, lib): + return self.library_filename(lib) - def find_library_file (self, dirs, lib, debug=0): + def find_library_file(self, dirs, lib, debug=0): # Prefer a debugging library if found (and requested), but deal # with it if we don't have one. if debug: @@ -573,8 +711,6 @@ # Oops, didn't find it in *any* of 'dirs' return None - # find_library_file () - # Helper methods for using the MSVC registry settings def find_exe(self, exe): @@ -586,67 +722,15 @@ absolute path that is known to exist. If none of them work, just return the original program name, 'exe'. """ - for p in self.__paths: fn = os.path.join(os.path.abspath(p), exe) if os.path.isfile(fn): return fn # didn't find it; try existing path - for p in string.split(os.environ['Path'],';'): + for p in os.environ['Path'].split(';'): fn = os.path.join(os.path.abspath(p),exe) if os.path.isfile(fn): return fn return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return string.split(self.__macros.sub(d[path]), ";") - else: - return string.split(d[path], ";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = string.join(p, ';') Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/spawn.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/spawn.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/spawn.py Wed Apr 21 19:49:59 2010 @@ -69,7 +69,6 @@ dry_run=0): executable = cmd[0] - cmd = _nt_quote_args(cmd) if search_path: # either we find one or it stays the same executable = find_executable(executable) or executable @@ -77,7 +76,8 @@ if not dry_run: # spawn for NT requires a full path to the .exe try: - rc = os.spawnv(os.P_WAIT, executable, cmd) + import subprocess + rc = subprocess.call(cmd) except OSError, exc: # this seems to happen when the command isn't found raise DistutilsExecError, \ Modified: pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/cpython-extension/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Wed Apr 21 19:49:59 2010 @@ -63,6 +63,7 @@ """Initialize the module as appropriate for NT""" g = {} g['EXE'] = ".exe" + g['SO'] = ".pyd" global _config_vars _config_vars = g From afa at codespeak.net Wed Apr 21 19:55:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 19:55:57 +0200 (CEST) Subject: [pypy-svn] r73950 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100421175557.2D28A282B9E@codespeak.net> Author: afa Date: Wed Apr 21 19:55:55 2010 New Revision: 73950 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Log: Argh, bad capitalization: PyNumber_Inplace -> PyNumber_InPlace Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Wed Apr 21 19:55:55 2010 @@ -47,7 +47,7 @@ def make_inplace_numbermethod(name, spacemeth): spacemeth = 'inplace_' + spacemeth.rstrip('_') @cpython_api([PyObject, PyObject], PyObject) - @func_rename('PyNumber_Inplace%s' % (name,)) + @func_rename('PyNumber_InPlace%s' % (name,)) def PyNumber_Method(space, w_o1, w_o2): meth = getattr(space, spacemeth) return meth(w_o1, w_o2) @@ -83,9 +83,9 @@ return space.pow(w_o1, w_o2, w_o3) @cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyNumber_InplacePower(space, w_o1, w_o2, w_o3): +def PyNumber_InPlacePower(space, w_o1, w_o2, w_o3): if not space.is_w(w_o3, space.w_None): raise OperationError(space.w_ValueError, space.wrap( - "PyNumber_InplacePower with non-None modulus is not supported")) + "PyNumber_InPlacePower with non-None modulus is not supported")) return space.inplace_pow(w_o1, w_o2) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Wed Apr 21 19:55:55 2010 @@ -19,7 +19,7 @@ assert api.PyObject_Size(w_l2) == 9 assert api.PyObject_Size(w_l) == 3 - w_l3 = api.PyNumber_InplaceMultiply(w_l, space.wrap(3)) + w_l3 = api.PyNumber_InPlaceMultiply(w_l, space.wrap(3)) assert api.PyObject_Size(w_l) == 9 assert w_l3 is w_l @@ -32,4 +32,4 @@ assert 4 == space.unwrap( api.PyNumber_Power(space.wrap(3), space.wrap(2), space.wrap(5))) assert 9 == space.unwrap( - api.PyNumber_InplacePower(space.wrap(3), space.wrap(2), space.w_None)) + api.PyNumber_InPlacePower(space.wrap(3), space.wrap(2), space.w_None)) From afa at codespeak.net Wed Apr 21 23:46:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Apr 2010 23:46:34 +0200 (CEST) Subject: [pypy-svn] r73951 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100421214634.AB0C2282B9E@codespeak.net> Author: afa Date: Wed Apr 21 23:46:32 2010 New Revision: 73951 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: Don't define these macros for now Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Wed Apr 21 23:46:32 2010 @@ -51,9 +51,11 @@ } \ } while (0) +#if 0 /* This will be added with python 2.6 */ #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#endif /* This will be added with python 2.6 */ #define Py_None (&_Py_NoneStruct) From afa at codespeak.net Thu Apr 22 00:07:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 00:07:59 +0200 (CEST) Subject: [pypy-svn] r73952 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100421220759.ECAC9282B9E@codespeak.net> Author: afa Date: Thu Apr 22 00:07:58 2010 New Revision: 73952 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Log: Oops, I broke compilation Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Thu Apr 22 00:07:58 2010 @@ -11,7 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBuffer_Type; -#define PyBuffer_Check(op) (Py_TYPE(op) == &PyBuffer_Type) +#define PyBuffer_Check(op) (((PyObject*)(ob))->ob_type == &PyBuffer_Type) #define Py_END_OF_BUFFER (-1) From afa at codespeak.net Thu Apr 22 00:10:56 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 00:10:56 +0200 (CEST) Subject: [pypy-svn] r73953 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100421221056.F1656282B9E@codespeak.net> Author: afa Date: Thu Apr 22 00:10:55 2010 New Revision: 73953 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Log: Typo again Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h Thu Apr 22 00:10:55 2010 @@ -11,7 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBuffer_Type; -#define PyBuffer_Check(op) (((PyObject*)(ob))->ob_type == &PyBuffer_Type) +#define PyBuffer_Check(op) (((PyObject*)(op))->ob_type == &PyBuffer_Type) #define Py_END_OF_BUFFER (-1) From arigo at codespeak.net Thu Apr 22 09:52:58 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 09:52:58 +0200 (CEST) Subject: [pypy-svn] r73954 - in pypy/branch/asmgcc-64/pypy: rpython/memory/gctransform translator/c translator/c/src Message-ID: <20100422075258.412D2282BEF@codespeak.net> Author: arigo Date: Thu Apr 22 09:52:55 2010 New Revision: 73954 Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/asmgcc-64/pypy/translator/c/gc.py pypy/branch/asmgcc-64/pypy/translator/c/src/mem.h Log: Check-in intermediate changes. Modified: pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/asmgcc-64/pypy/rpython/memory/gctransform/asmgcroot.py Thu Apr 22 09:52:55 2010 @@ -28,21 +28,11 @@ return livevars def pop_roots(self, hop, livevars): - if not livevars: - return - # mark the values as gc roots - for var in livevars: - v_adr = gen_cast(hop.llops, llmemory.Address, var) - v_newaddr = hop.genop("direct_call", [c_asm_gcroot, v_adr], - resulttype=llmemory.Address) - hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) + hop.genop("gc_asmgcroot", livevars) def build_root_walker(self): return AsmStackRootWalker(self) - def mark_call_cannotcollect(self, hop, name): - hop.genop("direct_call", [c_asm_nocollect, name]) - def gct_direct_call(self, hop): fnptr = hop.spaceop.args[0].value try: @@ -483,19 +473,6 @@ c_asm_stackwalk = Constant(pypy_asm_stackwalk, lltype.typeOf(pypy_asm_stackwalk)) -pypy_asm_gcroot = rffi.llexternal('pypy_asm_gcroot', - [llmemory.Address], - llmemory.Address, - sandboxsafe=True, - _nowrapper=True) -c_asm_gcroot = Constant(pypy_asm_gcroot, lltype.typeOf(pypy_asm_gcroot)) - -pypy_asm_nocollect = rffi.llexternal('pypy_asm_gc_nocollect', - [rffi.CCHARP], lltype.Void, - sandboxsafe=True, - _nowrapper=True) -c_asm_nocollect = Constant(pypy_asm_nocollect, lltype.typeOf(pypy_asm_nocollect)) - QSORT_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address, llmemory.Address], rffi.INT)) qsort = rffi.llexternal('qsort', Modified: pypy/branch/asmgcc-64/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/gc.py (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/gc.py Thu Apr 22 09:52:55 2010 @@ -326,12 +326,30 @@ def convert_weakref_to(self, ptarget): return framework.convert_weakref_to(ptarget) - def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): - if isinstance(op.args[1], Constant): - return '/* %s */' % (op,) - else: - args = [funcgen.expr(v) for v in op.args] - return '%s = %s; /* for moving GCs */' % (args[1], args[0]) + def OP_GC_ASMGCROOT(self, funcgen, op): + # The following pseudo-instruction is used by --gcrootfinder=asmgcc + # just after a call to tell gcc to put a GCROOT mark on each gc-pointer + # local variable. In practice the asm statement is often a no-op + # in the final machine code and doesn't prevent most optimizations, + # except that so far it forces its arguments to be in memory (instead + # of in the registers that are preserved by the call). + lines = [] + regs = [] + def generate(end=True): + bodylines = ['GCROOT %' + str(i) for i in range(len(regs))] + if end: + bodylines.append('GCREND') + lines.append('asm volatile ("%s" : :%s : "memory");' % ( + '\\n\\t'.join(bodylines), ','.join(regs))) + del regs[:] + # + for v in op.args: + if not isinstance(v, Constant): + if len(regs) == 8: + generate(end=False) + regs.append(' "m"(%s)' % funcgen.expr(v)) + generate() + return ''.join(lines) def common_gcheader_definition(self, defnode): return defnode.db.gctransformer.gc_fields() Modified: pypy/branch/asmgcc-64/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/asmgcc-64/pypy/translator/c/src/mem.h (original) +++ pypy/branch/asmgcc-64/pypy/translator/c/src/mem.h Thu Apr 22 09:52:55 2010 @@ -10,33 +10,6 @@ extern long pypy_asm_stackwalk(void*); #define __gcnoreorderhack __gcmapend -/* The following pseudo-instruction is used by --gcrootfinder=asmgcc - just after a call to tell gcc to put a GCROOT mark on each gc-pointer - local variable. All such local variables need to go through a "v = - pypy_asm_gcroot(v)". The old value should not be used any more by - the C code; this prevents the following case from occurring: gcc - could make two copies of the local variable (e.g. one in the stack - and one in a register), pass one to GCROOT, and later use the other - one. In practice the pypy_asm_gcroot() is often a no-op in the final - machine code and doesn't prevent most optimizations. */ - -/* With gcc, getting the asm() right was tricky, though. The asm() is - not volatile so that gcc is free to delete it if the output variable - is not used at all. We need to prevent gcc from moving the asm() - *before* the call that could cause a collection; this is the purpose - of the (unused) __gcnoreorderhack input argument. Any memory input - argument would have this effect: as far as gcc knows the call - instruction can modify arbitrary memory, thus creating the order - dependency that we want. */ - -#define pypy_asm_gcroot(p) ({void*_r; \ - asm ("/* GCROOT %0 */" : "=g" (_r) : \ - "0" (p), "m" (__gcnoreorderhack)); \ - _r; }) - -#define pypy_asm_gc_nocollect(f) asm volatile ("/* GC_NOCOLLECT " #f " */" \ - : : ) - #define pypy_asm_keepalive(v) asm volatile ("/* keepalive %0 */" : : \ "g" (v)) From arigo at codespeak.net Thu Apr 22 10:06:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 10:06:34 +0200 (CEST) Subject: [pypy-svn] r73955 - in pypy/branch/cpython-extension/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20100422080634.19FEC282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 10:06:32 2010 New Revision: 73955 Added: pypy/branch/cpython-extension/pypy/rlib/_rweakkeydict.py - copied, changed from r73882, pypy/trunk/pypy/rlib/rweakrefimpl.py pypy/branch/cpython-extension/pypy/rlib/_rweakvaldict.py - copied, changed from r73882, pypy/trunk/pypy/rlib/rweakrefimpl.py pypy/branch/cpython-extension/pypy/rlib/test/test_rweakkeydict.py (contents, props changed) pypy/branch/cpython-extension/pypy/rlib/test/test_rweakvaldict.py - copied, changed from r73882, pypy/trunk/pypy/rlib/test/test_rweakref.py Removed: pypy/branch/cpython-extension/pypy/rlib/rweakrefimpl.py pypy/branch/cpython-extension/pypy/rlib/test/test_rweakref.py Modified: pypy/branch/cpython-extension/pypy/rlib/rweakref.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rdict.py Log: Add an RPython variant of WeakKeyDictionary. Currently limited to value objects that don't have references to too many other objects (see comment in rlib/_rweakkeydict.py). Copied: pypy/branch/cpython-extension/pypy/rlib/_rweakkeydict.py (from r73882, pypy/trunk/pypy/rlib/rweakrefimpl.py) ============================================================================== --- pypy/trunk/pypy/rlib/rweakrefimpl.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/_rweakkeydict.py Thu Apr 22 10:06:32 2010 @@ -1,44 +1,46 @@ from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass, rdict +from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr -from pypy.rlib.rweakref import RWeakValueDictionary +from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rlib import jit +from pypy.rlib.objectmodel import compute_identity_hash +from pypy.rlib.objectmodel import we_are_translated -class WeakValueDictRepr(Repr): +# Warning: this implementation of RWeakKeyDictionary is not exactly +# leaking, but can keep around some values for a long time, even after +# the corresponding keys were freed. They will be eventually freed if +# you continue to manipulate the dictionary. Avoid to use this if the +# values are objects that might keep alive tons of memory. + + +class WeakKeyDictRepr(Repr): def __init__(self, rtyper): self.rtyper = rtyper self.lowleveltype = lltype.Ptr(WEAKDICT) self.dict_cache = {} def convert_const(self, weakdict): - if not isinstance(weakdict, RWeakValueDictionary): - raise TyperError("expected an RWeakValueDictionary: %r" % ( + if not isinstance(weakdict, RWeakKeyDictionary): + raise TyperError("expected an RWeakKeyDictionary: %r" % ( weakdict,)) try: key = Constant(weakdict) return self.dict_cache[key] except KeyError: self.setup() + if weakdict.length() != 0: + raise TyperError("got a non-empty prebuilt RWeakKeyDictionary") l_dict = ll_new_weakdict() self.dict_cache[key] = l_dict - bk = self.rtyper.annotator.bookkeeper - classdef = bk.getuniqueclassdef(weakdict._valueclass) - r_key = rstr.string_repr - r_value = getinstancerepr(self.rtyper, classdef) - for dictkey, dictvalue in weakdict._dict.items(): - llkey = r_key.convert_const(dictkey) - llvalue = r_value.convert_const(dictvalue) - if llvalue: - llvalue = lltype.cast_pointer(rclass.OBJECTPTR, llvalue) - ll_set_nonnull(l_dict, llkey, llvalue) return l_dict def rtype_method_get(self, hop): - v_d, v_key = hop.inputargs(self, rstr.string_repr) + r_object = getinstancerepr(self.rtyper, None) + v_d, v_key = hop.inputargs(self, r_object) hop.exception_cannot_occur() v_result = hop.gendirectcall(ll_get, v_d, v_key) v_result = hop.genop("cast_pointer", [v_result], @@ -46,16 +48,19 @@ return v_result def rtype_method_set(self, hop): - v_d, v_key, v_value = hop.inputargs(self, rstr.string_repr, - hop.args_r[2]) + r_object = getinstancerepr(self.rtyper, None) + v_d, v_key, v_value = hop.inputargs(self, r_object, r_object) hop.exception_cannot_occur() if hop.args_s[2].is_constant() and hop.args_s[2].const is None: hop.gendirectcall(ll_set_null, v_d, v_key) else: - v_value = hop.genop("cast_pointer", [v_value], - resulttype=rclass.OBJECTPTR) hop.gendirectcall(ll_set, v_d, v_key, v_value) + def rtype_method_length(self, hop): + v_d, = hop.inputargs(self) + hop.exception_cannot_occur() + return hop.gendirectcall(ll_length, v_d) + def specialize_make_weakdict(hop): hop.exception_cannot_occur() @@ -65,33 +70,47 @@ # ____________________________________________________________ +NULLVALUE = lltype.nullptr(rclass.OBJECTPTR.TO) WEAKDICTENTRY = lltype.Struct("weakdictentry", - ("key", lltype.Ptr(rstr.STR)), - ("value", llmemory.WeakRefPtr)) + ("key", llmemory.WeakRefPtr), + ("value", rclass.OBJECTPTR), + ("f_hash", lltype.Signed)) + +def ll_debugrepr(x): + if x: + h = compute_identity_hash(x) + else: + h = 0 + return '<%x>' % (h,) def ll_valid(entries, i): - return (bool(entries[i].value) and - bool(weakref_deref(rclass.OBJECTPTR, entries[i].value))) + key = entries[i].key + if not key: + return False + elif weakref_deref(rclass.OBJECTPTR, key): + return True + else: + # The entry might be a dead weakref still holding a strong + # reference to the value; for this case, we clear the old + # value from the entry, if any. + entries[i].value = NULLVALUE + return False def ll_everused(entries, i): - return bool(entries[i].value) - -def ll_hash(entries, i): - return str_fasthashfn(entries[i].key) -str_fasthashfn = rstr.string_repr.get_ll_fasthash_function() + return bool(entries[i].key) entrymeths = { 'allocate': lltype.typeMethod(rdict._ll_malloc_entries), 'delete': rdict._ll_free_entries, 'valid': ll_valid, 'everused': ll_everused, - 'hash': ll_hash, + 'hash': rdict.ll_hash_from_cache, + 'no_direct_compare': True, } WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, - hints={'weakarray': 'value'}) - -ll_strhash = rstr.LLHelpers.ll_strhash + hints={'weakarray': 'key'}) +# NB. the 'hints' is not used so far ^^^ @jit.dont_look_inside def ll_new_weakdict(): @@ -103,14 +122,15 @@ @jit.dont_look_inside def ll_get(d, llkey): - hash = ll_strhash(llkey) + hash = compute_identity_hash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) - #llop.debug_print(lltype.Void, i, 'get') - valueref = d.entries[i].value - if valueref: - return weakref_deref(rclass.OBJECTPTR, valueref) - else: - return lltype.nullptr(rclass.OBJECTPTR.TO) + #llop.debug_print(lltype.Void, i, 'get', hex(hash), + # ll_debugrepr(d.entries[i].key), + # ll_debugrepr(d.entries[i].value)) + # NB. ll_valid() above was just called at least on entry i, so if + # it is an invalid entry with a dead weakref, the value was reset + # to NULLVALUE. + return d.entries[i].value @jit.dont_look_inside def ll_set(d, llkey, llvalue): @@ -121,51 +141,70 @@ @jit.dont_look_inside def ll_set_nonnull(d, llkey, llvalue): - hash = ll_strhash(llkey) - valueref = weakref_create(llvalue) # GC effects here, before the rest + hash = compute_identity_hash(llkey) + keyref = weakref_create(llkey) # GC effects here, before the rest i = rdict.ll_dict_lookup(d, llkey, hash) everused = d.entries.everused(i) - d.entries[i].key = llkey - d.entries[i].value = valueref - #llop.debug_print(lltype.Void, i, 'stored') + d.entries[i].key = keyref + d.entries[i].value = llvalue + d.entries[i].f_hash = hash + #llop.debug_print(lltype.Void, i, 'stored', hex(hash), + # ll_debugrepr(llkey), + # ll_debugrepr(llvalue)) if not everused: d.num_pristine_entries -= 1 - if d.num_pristine_entries <= len(d.entries) / 3: + if d.num_pristine_entries * 3 <= len(d.entries): #llop.debug_print(lltype.Void, 'RESIZE') ll_weakdict_resize(d) @jit.dont_look_inside def ll_set_null(d, llkey): - hash = ll_strhash(llkey) + hash = compute_identity_hash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) if d.entries.everused(i): # If the entry was ever used, clean up its key and value. # We don't store a NULL value, but a dead weakref, because # the entry must still be marked as everused(). - d.entries[i].value = llmemory.dead_wref - d.entries[i].key = lltype.nullptr(rstr.STR) + d.entries[i].key = llmemory.dead_wref + d.entries[i].value = NULLVALUE #llop.debug_print(lltype.Void, i, 'zero') -def ll_weakdict_resize(d): - # first set num_items to its correct, up-to-date value +def ll_update_num_items(d): entries = d.entries num_items = 0 for i in range(len(entries)): if entries.valid(i): num_items += 1 d.num_items = num_items + +def ll_weakdict_resize(d): + # first set num_items to its correct, up-to-date value + ll_update_num_items(d) rdict.ll_dict_resize(d) -str_keyeq = lltype.staticAdtMethod(rstr.string_repr.get_ll_eq_function()) +def ll_keyeq(d, weakkey1, realkey2): + # only called by ll_dict_lookup() with the first arg coming from an + # entry.key, and the 2nd arg being the argument to ll_dict_lookup(). + if not weakkey1: + assert bool(realkey2) + return False + return weakref_deref(rclass.OBJECTPTR, weakkey1) == realkey2 + + at jit.dont_look_inside +def ll_length(d): + # xxx slow, but it's only for debugging + ll_update_num_items(d) + #llop.debug_print(lltype.Void, 'length:', d.num_items) + return d.num_items dictmeths = { 'll_get': ll_get, 'll_set': ll_set, - 'keyeq': str_keyeq, + 'keyeq': ll_keyeq, 'paranoia': False, } -WEAKDICT = lltype.GcStruct("weakdict", +WEAKDICT = lltype.GcStruct("weakkeydict", ("num_items", lltype.Signed), ("num_pristine_entries", lltype.Signed), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), Copied: pypy/branch/cpython-extension/pypy/rlib/_rweakvaldict.py (from r73882, pypy/trunk/pypy/rlib/rweakrefimpl.py) ============================================================================== --- pypy/trunk/pypy/rlib/rweakrefimpl.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/_rweakvaldict.py Thu Apr 22 10:06:32 2010 @@ -46,14 +46,13 @@ return v_result def rtype_method_set(self, hop): + r_object = getinstancerepr(self.rtyper, None) v_d, v_key, v_value = hop.inputargs(self, rstr.string_repr, - hop.args_r[2]) + r_object) hop.exception_cannot_occur() if hop.args_s[2].is_constant() and hop.args_s[2].const is None: hop.gendirectcall(ll_set_null, v_d, v_key) else: - v_value = hop.genop("cast_pointer", [v_value], - resulttype=rclass.OBJECTPTR) hop.gendirectcall(ll_set, v_d, v_key, v_value) @@ -70,8 +69,8 @@ ("value", llmemory.WeakRefPtr)) def ll_valid(entries, i): - return (bool(entries[i].value) and - bool(weakref_deref(rclass.OBJECTPTR, entries[i].value))) + value = entries[i].value + return bool(value) and bool(weakref_deref(rclass.OBJECTPTR, value)) def ll_everused(entries, i): return bool(entries[i].value) @@ -90,6 +89,7 @@ WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, hints={'weakarray': 'value'}) +# NB. the 'hints' is not used so far ^^^ ll_strhash = rstr.LLHelpers.ll_strhash @@ -130,7 +130,7 @@ #llop.debug_print(lltype.Void, i, 'stored') if not everused: d.num_pristine_entries -= 1 - if d.num_pristine_entries <= len(d.entries) / 3: + if d.num_pristine_entries * 3 <= len(d.entries): #llop.debug_print(lltype.Void, 'RESIZE') ll_weakdict_resize(d) @@ -165,7 +165,7 @@ 'paranoia': False, } -WEAKDICT = lltype.GcStruct("weakdict", +WEAKDICT = lltype.GcStruct("weakvaldict", ("num_items", lltype.Signed), ("num_pristine_entries", lltype.Signed), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), Modified: pypy/branch/cpython-extension/pypy/rlib/rweakref.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/rweakref.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/rweakref.py Thu Apr 22 10:06:32 2010 @@ -1,6 +1,7 @@ """ Weakref support in RPython. Supports ref() without callbacks, -and a limited version of WeakValueDictionary. LLType only for now! +a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. +LLType only for now! """ import weakref @@ -27,6 +28,36 @@ self._dict[key] = value +class RWeakKeyDictionary(object): + """A dictionary containing weak keys. + Keys and values must be instances. + Prebuilt RWeakKeyDictionaries must be empty. + """ + + def __init__(self, keyclass, valueclass): + self._dict = weakref.WeakKeyDictionary() + self._keyclass = keyclass + self._valueclass = valueclass + + def get(self, key): + """Get the value associated to 'key', or None by default.""" + assert isinstance(key, self._keyclass) + return self._dict.get(key, None) + + def set(self, key, value): + """Set the key/value pair (or delete it if value is None).""" + assert isinstance(key, self._keyclass) + if value is None: + self._dict.pop(key, None) + else: + assert isinstance(value, self._valueclass) + self._dict[key] = value + + def length(self): + """Mostly for debugging. Slow, don't use in real code.""" + return len(self._dict) + + # ____________________________________________________________ from pypy.rpython import extregistry @@ -41,8 +72,8 @@ self.valueclassdef = valueclassdef def rtyper_makerepr(self, rtyper): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.WeakValueDictRepr(rtyper) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.WeakValueDictRepr(rtyper) def rtyper_makekey_ex(self, rtyper): return self.__class__, @@ -65,14 +96,11 @@ _about_ = RWeakValueDictionary def compute_result_annotation(self, s_valueclass): - assert isinstance(s_valueclass, annmodel.SomePBC) - assert s_valueclass.is_constant() - [desc] = s_valueclass.descriptions - return SomeWeakValueDict(desc.getuniqueclassdef()) + return SomeWeakValueDict(_getclassdef(s_valueclass)) def specialize_call(self, hop): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.specialize_make_weakdict(hop) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.specialize_make_weakdict(hop) class Entry(extregistry.ExtRegistryEntry): _type_ = RWeakValueDictionary @@ -81,3 +109,65 @@ bk = self.bookkeeper x = self.instance return SomeWeakValueDict(bk.getuniqueclassdef(x._valueclass)) + +def _getclassdef(s_instance): + assert isinstance(s_instance, annmodel.SomePBC) + assert s_instance.is_constant() + [desc] = s_instance.descriptions + return desc.getuniqueclassdef() + +# ____________________________________________________________ + +class SomeWeakKeyDict(annmodel.SomeObject): + knowntype = RWeakKeyDictionary + + def __init__(self, keyclassdef, valueclassdef): + self.keyclassdef = keyclassdef + self.valueclassdef = valueclassdef + + def rtyper_makerepr(self, rtyper): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.WeakKeyDictRepr(rtyper) + + def rtyper_makekey_ex(self, rtyper): + return self.__class__, + + def method_get(self, s_key): + assert isinstance(s_key, annmodel.SomeInstance) + assert s_key.classdef.issubclass(self.keyclassdef) + return annmodel.SomeInstance(self.valueclassdef, can_be_None=True) + + def method_set(self, s_key, s_value): + s_oldvalue = self.method_get(s_key) + assert s_oldvalue.contains(s_value) + + def method_length(self): + return annmodel.SomeInteger(nonneg=True) + +class __extend__(pairtype(SomeWeakKeyDict, SomeWeakKeyDict)): + def union((s_wkd1, s_wkd2)): + if s_wkd1.keyclassdef is not s_wkd2.keyclassdef: + return SomeObject() # not the same key class! complain... + if s_wkd1.valueclassdef is not s_wkd2.valueclassdef: + return SomeObject() # not the same value class! complain... + return SomeWeakKeyDict(s_wkd1.keyclassdef, s_wkd1.valueclassdef) + +class Entry(extregistry.ExtRegistryEntry): + _about_ = RWeakKeyDictionary + + def compute_result_annotation(self, s_keyclass, s_valueclass): + return SomeWeakKeyDict(_getclassdef(s_keyclass), + _getclassdef(s_valueclass)) + + def specialize_call(self, hop): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.specialize_make_weakdict(hop) + +class Entry(extregistry.ExtRegistryEntry): + _type_ = RWeakKeyDictionary + + def compute_annotation(self): + bk = self.bookkeeper + x = self.instance + return SomeWeakKeyDict(bk.getuniqueclassdef(x._keyclass), + bk.getuniqueclassdef(x._valueclass)) Added: pypy/branch/cpython-extension/pypy/rlib/test/test_rweakkeydict.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/rlib/test/test_rweakkeydict.py Thu Apr 22 10:06:32 2010 @@ -0,0 +1,137 @@ +import py +from pypy.rlib import rgc +from pypy.rlib.rweakref import RWeakKeyDictionary +from pypy.rpython.test.test_llinterp import interpret + +class KX(object): + pass + +class KY(KX): + pass + +class VX(object): + pass + +class VY(VX): + pass + + +def make_test(loop=100, prebuilt=None): + def g(d): + assert d.get(KX()) is None + assert d.get(KY()) is None + k1 = KX(); k2 = KX(); k3 = KX() + v1 = VX(); v2 = VX(); v3 = VX() + d.set(k1, v1) + d.set(k2, v2) + d.set(k3, v3) + assert d.get(k1) is v1 + assert d.get(k2) is v2 + assert d.get(k3) is v3 + assert d.get(KX()) is None + assert d.length() == 3 + return k1, k3, v1, v2, v3 # k2 dies + def f(): + d = prebuilt + if d is None: + d = RWeakKeyDictionary(KX, VX) + k1, k3, v1, v2, v3 = g(d) + rgc.collect(); rgc.collect() + assert d.get(k1) is v1 + assert d.get(k3) is v3 + assert d.get(k1) is not v2 + assert d.get(k3) is not v2 + assert d.length() == 2 + d.set(k1, None) + assert d.get(k1) is None + assert d.get(k3) is v3 + assert d.length() == 1 + # resizing should also work + lots_of_keys = [KX() for i in range(loop)] + for k in lots_of_keys: + d.set(k, v1) + for k in lots_of_keys: + assert d.get(k) is v1 + assert d.get(k1) is None + assert d.get(k3) is v3 + assert d.length() == loop + 1 + # a subclass + ky = KY() + vy = VY() + d.set(ky, vy) + assert d.get(ky) is vy + assert d.length() == loop + 2 + # deleting by storing Nones + for k in lots_of_keys: + d.set(k, None) + for k in lots_of_keys: + assert d.get(k) is None + assert d.get(k1) is None + assert d.get(k3) is v3 + assert d.get(ky) is vy + assert d.length() == 2 + return f + +def test_RWeakKeyDictionary(): + make_test()() + +def test_rpython_RWeakKeyDictionary(): + interpret(make_test(loop=12), []) + +def test_rpython_prebuilt(): + f = make_test(loop=12, prebuilt=RWeakKeyDictionary(KX, VX)) + interpret(f, []) + +def test_rpython_merge_RWeakKeyDictionary(): + empty = RWeakKeyDictionary(KX, VX) + def f(n): + k = KX() + v = VX() + if n: + d = empty + else: + d = RWeakKeyDictionary(KX, VX) + d.set(k, v) + return d.get(k) is v + assert f(0) + assert interpret(f, [0]) + assert not f(1) + assert not interpret(f, [1]) + + +def test_rpython_merge_RWeakKeyDictionary2(): + class A(object): + def __init__(self): + self.d = RWeakKeyDictionary(KX, A) + def f(self, key): + a = A() + self.d.set(key, a) + return a + empty = A() + def f(x): + a = A() + if x: + a = empty + k = KX() + a2 = a.f(k) + assert a.d.get(k) is a2 + f(0) + interpret(f, [0]) + f(1) + interpret(f, [1]) + + def g(x): + if x: + d = RWeakKeyDictionary(KX, VX) + else: + d = RWeakKeyDictionary(KY, VX) + d.set(KX(), VX()) + py.test.raises(Exception, interpret, g, [1]) + + def g(x): + if x: + d = RWeakKeyDictionary(KX, VX) + else: + d = RWeakKeyDictionary(KX, VY) + d.set(KX(), VX()) + py.test.raises(Exception, interpret, g, [1]) Copied: pypy/branch/cpython-extension/pypy/rlib/test/test_rweakvaldict.py (from r73882, pypy/trunk/pypy/rlib/test/test_rweakref.py) ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rweakref.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/test/test_rweakvaldict.py Thu Apr 22 10:06:32 2010 @@ -130,5 +130,4 @@ else: d = RWeakValueDictionary(Y) d.set("x", X()) - return 41 py.test.raises(Exception, interpret, g, [1]) Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rdict.py Thu Apr 22 10:06:32 2010 @@ -508,12 +508,14 @@ def ll_dict_lookup(d, key, hash): entries = d.entries + ENTRIES = lltype.typeOf(entries).TO + direct_compare = not hasattr(ENTRIES, 'no_direct_compare') mask = len(entries) - 1 i = hash & mask # do the first try before any looping if entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i # found the entry if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -548,7 +550,7 @@ return freeslot elif entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to From arigo at codespeak.net Thu Apr 22 11:08:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 11:08:10 +0200 (CEST) Subject: [pypy-svn] r73956 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100422090810.AB84C282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 11:08:08 2010 New Revision: 73956 Added: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py - copied unchanged from r73939, pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py Removed: pypy/branch/blackhole-improvement/pypy/jit/codewriter/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/codewriter.py Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: Start integration: kill metainterp/codewriter.py and move blackhole.py into metainterp/. Almost all tests fail so far. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Thu Apr 22 11:08:08 2010 @@ -1,5 +1,4 @@ from pypy.jit.codewriter.codewriter import CodeWriter -from pypy.jit.codewriter.blackhole import BlackholeInterpreter def test_loop(): @@ -22,6 +21,7 @@ 'int_return/i': 4} def test_integration(): + from pypy.jit.metainterp.blackhole import BlackholeInterpreter def f(a, b): while a > 2: b += a Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 22 11:08:08 2010 @@ -8,7 +8,7 @@ from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp import codewriter, executor +from pypy.jit.metainterp import executor from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE @@ -183,23 +183,6 @@ # ------------------------------ - for _n in range(codewriter.MAX_MAKE_NEW_VARS): - _decl = ', '.join(["'box'" for _i in range(_n)]) - _allargs = ', '.join(["box%d" % _i for _i in range(_n)]) - exec py.code.Source(""" - @arguments(%s) - def opimpl_make_new_vars_%d(self, %s): - if not we_are_translated(): - check_args(%s) - self.env = [%s] - """ % (_decl, _n, _allargs, _allargs, _allargs)).compile() - - @arguments("varargs") - def opimpl_make_new_vars(self, newenv): - if not we_are_translated(): - check_args(*newenv) - self.env = newenv - for _opimpl in ['int_add', 'int_sub', 'int_mul', 'int_floordiv', 'int_mod', 'int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu Apr 22 11:08:08 2010 @@ -4,7 +4,7 @@ from pypy.rlib.jit import OPTIMIZER_FULL, OPTIMIZER_SIMPLE, loop_invariant from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner -from pypy.jit.metainterp import codewriter, pyjitpl, history +from pypy.jit.metainterp import pyjitpl, history from pypy.jit.metainterp.policy import JitPolicy, StopAtXPolicy from pypy.jit.codewriter import support from pypy import conftest Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Thu Apr 22 11:08:08 2010 @@ -1,8 +1,67 @@ from pypy.rlib.jit import JitDriver from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.jit.metainterp import pyjitpl +from pypy.jit.metainterp.blackhole import BlackholeInterpreter +from pypy.jit.codewriter.assembler import JitCode +def test_simple(): + jitcode = JitCode("test") + jitcode.setup("\x00\x00\x01\x02" + "\x01\x02", + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_add/iii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(0, 40) + blackholeinterp.setarg_i(1, 2) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + +def test_simple_const(): + jitcode = JitCode("test") + jitcode.setup("\x00\x30\x01\x02" + "\x01\x02", + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_sub/cii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(1, 6) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + +def test_simple_bigconst(): + jitcode = JitCode("test") + jitcode.setup("\x00\xFD\x01\x02" + "\x01\x02", + [666, 666, 10042, 666]) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'int_sub/iii': 0, + 'int_return/i': 1}) + blackholeinterp.setarg_i(1, 10000) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + +def test_simple_loop(): + jitcode = JitCode("test") + jitcode.setup("\x00\x10\x00\x16\x02" # L1: goto_if_not_int_gt L2, %i0, 2 + "\x01\x17\x16\x17" # int_add %i1, %i0, %i1 + "\x02\x16\x01\x16" # int_sub %i0, $1, %i0 + "\x03\x00\x00" # goto L1 + "\x04\x17", # L2: int_return %i1 + []) + blackholeinterp = BlackholeInterpreter() + blackholeinterp.setup_insns({'goto_if_not_int_gt/Lic': 0, + 'int_add/iii': 1, + 'int_sub/ici': 2, + 'goto/L': 3, + 'int_return/i': 4}) + blackholeinterp.setarg_i(0x16, 6) # %i0 + blackholeinterp.setarg_i(0x17, 100) # %i1 + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 100+6+5+4+3 + +# ____________________________________________________________ + class BlackholeTests(object): def meta_interp(self, *args): @@ -10,6 +69,7 @@ previnit(frame, metainterp, jitcode, greenkey) self.seen_frames.append(jitcode.name) # + from pypy.jit.metainterp import pyjitpl previnit = pyjitpl.MIFrame.__init__.im_func try: self.seen_frames = [] Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Thu Apr 22 11:08:08 2010 @@ -16,7 +16,6 @@ from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.unsimplify import call_final_function -from pypy.jit.metainterp import codewriter from pypy.jit.metainterp import history, pyjitpl, gc from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.policy import JitPolicy From arigo at codespeak.net Thu Apr 22 11:47:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 11:47:26 +0200 (CEST) Subject: [pypy-svn] r73957 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100422094726.9BB77282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 11:47:24 2010 New Revision: 73957 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Two tests in test_basic pass again, this time running on the new blackhole interp. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Thu Apr 22 11:47:24 2010 @@ -3,11 +3,13 @@ from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.jitter import transform_graph +from pypy.jit.codewriter.format import format_assembler class CodeWriter(object): - def __init__(self): + def __init__(self, rtyper=None): + self.rtyper = rtyper self.assembler = Assembler() def transform_func_to_jitcode(self, func, values, type_system='lltype'): @@ -16,11 +18,22 @@ graph = rtyper.annotator.translator.graphs[0] return self.transform_graph_to_jitcode(graph) - def transform_graph_to_jitcode(self, graph): + def transform_graph_to_jitcode(self, graph, verbose=False): transform_graph(graph) regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) ssarepr = flatten_graph(graph, regallocs) + if verbose: + print graph + print indent(format_assembler(ssarepr), 4) jitcode = self.assembler.assemble(ssarepr) return jitcode + + def make_jitcodes(self, maingraph, verbose=False): + return self.transform_graph_to_jitcode(maingraph, verbose) + + +def indent(s, indent): + indent = ' ' * indent + return indent + s.replace('\n', '\n'+indent).rstrip(' ') Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 22 11:47:24 2010 @@ -26,7 +26,8 @@ class BlackholeInterpreter(object): - def __init__(self): + def __init__(self, cpu=None): + self.cpu = cpu self.registers_i = [MissingValue()] * 256 self.registers_r = [MissingValue()] * 256 self.registers_f = [MissingValue()] * 256 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu Apr 22 11:47:24 2010 @@ -6,15 +6,33 @@ from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp import pyjitpl, history from pypy.jit.metainterp.policy import JitPolicy, StopAtXPolicy -from pypy.jit.codewriter import support from pypy import conftest from pypy.rlib.rarithmetic import ovfcheck from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -def _get_bare_metainterp(func, values, CPUClass, type_system, - listops=False): +def _get_jitcodes(func, values, type_system): + from pypy.jit.codewriter import support, codewriter + + rtyper = support.annotate(func, values, type_system=type_system) + graphs = rtyper.annotator.translator.graphs + cw = codewriter.CodeWriter(rtyper) + mainjitcode = cw.make_jitcodes(graphs[0], verbose=True) + return cw, mainjitcode + +def _run_with_blackhole(CPUClass, cw, mainjitcode, args): + from pypy.jit.metainterp.blackhole import BlackholeInterpreter + stats = history.Stats() + cpu = CPUClass(cw.rtyper, stats, None, False) + blackholeinterp = BlackholeInterpreter(cpu) + blackholeinterp.setup_insns(cw.assembler.insns) + for i, value in enumerate(args): + blackholeinterp.setarg_i(i, value) + blackholeinterp.run(mainjitcode, 0) + return blackholeinterp.result_i + +def _get_bare_metainterp(func, values, CPUClass, type_system): from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation.model import lltype_to_annotation from pypy.rpython.test.test_llinterp import gengraph @@ -64,6 +82,14 @@ return ll_meta_interp(*args, **kwds) def interp_operations(self, f, args, **kwds): + # get the JitCodes for the function f + cw, mainjitcode = _get_jitcodes(f, args, self.type_system) + # try to run it with blackhole.py + result = _run_with_blackhole(self.CPUClass, cw, mainjitcode, args) + # try to run it with pyjitpl.py + # -- XXX --- missing + return result + from pypy.jit.metainterp import simple_optimize class DoneWithThisFrame(Exception): From arigo at codespeak.net Thu Apr 22 11:50:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 11:50:01 +0200 (CEST) Subject: [pypy-svn] r73958 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100422095001.6F199282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 11:50:00 2010 New Revision: 73958 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: A third test. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 22 11:50:00 2010 @@ -1,6 +1,8 @@ from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import intmask, LONG_BIT +from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint from pypy.tool.sourcetools import func_with_new_name +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop def arguments(*argtypes, **kwds): @@ -150,6 +152,11 @@ def opimpl_int_sub(self, a, b): return a - b + @arguments("i", "i", returns="i") + def opimpl_uint_floordiv(self, a, b): + c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) + return intmask(c) + @arguments("i") def opimpl_int_return(self, a): self.result_i = a From afa at codespeak.net Thu Apr 22 12:57:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 12:57:18 +0200 (CEST) Subject: [pypy-svn] r73959 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422105718.E63B2282B9E@codespeak.net> Author: afa Date: Thu Apr 22 12:57:17 2010 New Revision: 73959 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: still not the correct approach IMO, but at least reduce the number of warnings, and fix errors with C++ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 22 12:57:17 2010 @@ -626,7 +626,7 @@ typ = typ.replace("*", "") pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) if not globals_are_pointers and "#" not in name: - pypy_decls.append("#define %s &%s" % (name, name,)) + pypy_decls.append("#define %s (PyObject*)&%s" % (name, name,)) pypy_decls.append("#endif /*PYPY_STANDALONE*/\n") pypy_decl_h = udir.join('pypy_decl.h') From afa at codespeak.net Thu Apr 22 12:59:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 12:59:01 +0200 (CEST) Subject: [pypy-svn] r73960 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422105901.977F2282B9E@codespeak.net> Author: afa Date: Thu Apr 22 12:58:59 2010 New Revision: 73960 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Log: Add PyGILState_Ensure and PyGILState_Release, empty for the moment Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 22 12:58:59 2010 @@ -63,6 +63,10 @@ assert CONST_STRING is not rffi.CCHARP assert CONST_WSTRING is not rffi.CWCHARP +PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', + typedef='PyGILState_STATE', + compilation_info=CConfig._compilation_info_) + constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Thu Apr 22 12:58:59 2010 @@ -1,5 +1,5 @@ from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - cpython_struct + cpython_struct, PyGILState_STATE from pypy.rpython.lltypesystem import rffi, lltype @@ -28,3 +28,11 @@ from pypy.module.thread.gil import after_external_call after_external_call() + at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) +def PyGILState_Ensure(space): + return 0 + + at cpython_api([PyGILState_STATE], lltype.Void) +def PyGILState_Release(space, state): + return + From afa at codespeak.net Thu Apr 22 12:59:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 12:59:38 +0200 (CEST) Subject: [pypy-svn] r73961 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422105938.048EF282B9E@codespeak.net> Author: afa Date: Thu Apr 22 12:59:36 2010 New Revision: 73961 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Log: Really generate a void* for PyLong_FromVoidPtr. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Thu Apr 22 12:59:36 2010 @@ -66,7 +66,7 @@ pend[0] = rffi.ptradd(str, len(s)) return space.call_function(space.w_long, w_str, w_base) - at cpython_api([rffi.VOIDP], PyObject) + at cpython_api([rffi.VOIDP_real], PyObject) def PyLong_FromVoidPtr(space, p): """Create a Python integer or long integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). From afa at codespeak.net Thu Apr 22 13:00:33 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 13:00:33 +0200 (CEST) Subject: [pypy-svn] r73962 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422110033.41392282B9E@codespeak.net> Author: afa Date: Thu Apr 22 13:00:31 2010 New Revision: 73962 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Add _Py_NewReference. Not sure what it is used for. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 22 13:00:31 2010 @@ -277,6 +277,10 @@ if DEBUG_REFCOUNT: debug_refcount("INCREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) + at cpython_api([PyObject], lltype.Void) +def _Py_NewReference(space, obj): + obj.c_ob_refcnt = 1 + def _Py_Dealloc(space, obj): from pypy.module.cpyext.api import generic_cpy_call_dont_decref pto = obj.c_ob_type From afa at codespeak.net Thu Apr 22 13:01:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 13:01:59 +0200 (CEST) Subject: [pypy-svn] r73963 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100422110159.3333D282B9E@codespeak.net> Author: afa Date: Thu Apr 22 13:01:57 2010 New Revision: 73963 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Log: Should have been committed with r73960. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Thu Apr 22 13:01:57 2010 @@ -5,6 +5,10 @@ int initialized; // not used } PyThreadState; +typedef struct _is { + int _foo; +} PyInterpreterState; + #define Py_BEGIN_ALLOW_THREADS { \ PyThreadState *_save; \ _save = PyEval_SaveThread(); @@ -13,4 +17,8 @@ #define Py_END_ALLOW_THREADS PyEval_RestoreThread(_save); \ } +typedef + enum {PyGILState_LOCKED, PyGILState_UNLOCKED} + PyGILState_STATE; + #endif /* !Py_PYSTATE_H */ From afa at codespeak.net Thu Apr 22 13:06:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 13:06:15 +0200 (CEST) Subject: [pypy-svn] r73964 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422110615.C83D7282B9E@codespeak.net> Author: afa Date: Thu Apr 22 13:06:14 2010 New Revision: 73964 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Log: Add PyMapping_Check and PyMapping_Items Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Thu Apr 22 13:06:14 2010 @@ -1,9 +1,24 @@ -from pypy.module.cpyext.api import cpython_api +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL from pypy.module.cpyext.pyobject import PyObject + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyMapping_Check(space, w_obj): + """Return 1 if the object provides mapping protocol, and 0 otherwise. This + function always succeeds.""" + return int(space.findattr(w_obj, space.wrap("items")) is not None) + @cpython_api([PyObject], PyObject) def PyMapping_Keys(space, w_obj): """On success, return a list of the keys in object o. On failure, return NULL. This is equivalent to the Python expression o.keys().""" return space.call_method(w_obj, "keys") + + at cpython_api([PyObject], PyObject) +def PyMapping_Items(space, w_obj): + """On success, return a list of the items in object o, where each item is a tuple + containing a key-value pair. On failure, return NULL. This is equivalent to + the Python expression o.items().""" + return space.call_method(w_obj, "items") + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Thu Apr 22 13:06:14 2010 @@ -2,7 +2,16 @@ class TestMapping(BaseApiTest): + def test_check(self, space, api): + assert api.PyMapping_Check(space.newdict()) + assert not api.PyMapping_Check(space.newlist([])) + def test_keys(self, space, api): w_d = space.newdict() space.setitem(w_d, space.wrap("a"), space.wrap("a")) assert space.eq_w(api.PyMapping_Keys(w_d), space.wrap(["a"])) + + def test_items(self, space, api): + w_d = space.newdict() + space.setitem(w_d, space.wrap("a"), space.wrap("b")) + assert space.eq_w(api.PyMapping_Items(w_d), space.wrap([("a", "b")])) From afa at codespeak.net Thu Apr 22 13:07:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 13:07:06 +0200 (CEST) Subject: [pypy-svn] r73965 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422110706.1D50A282B9E@codespeak.net> Author: afa Date: Thu Apr 22 13:07:03 2010 New Revision: 73965 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Remove from stubs some recently added functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Thu Apr 22 13:07:03 2010 @@ -3015,42 +3015,6 @@ """ raise NotImplementedError - at cpython_api([], {PyGILState_STATE}) -def PyGILState_Ensure(space, ): - """Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the global interpreter lock. This may - be called as many times as desired by a thread as long as each call is - matched with a call to PyGILState_Release(). In general, other - thread-related APIs may be used between PyGILState_Ensure() and - PyGILState_Release() calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS macros is - acceptable. - - The return value is an opaque "handle" to the thread state when - PyGILState_Ensure() was called, and must be passed to - PyGILState_Release() to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles cannot be shared - each - unique call to PyGILState_Ensure() must save the handle for its call - to PyGILState_Release(). - - When the function returns, the current thread will hold the GIL. Failure is a - fatal error. - """ - raise NotImplementedError - - at cpython_api([{PyGILState_STATE}], lltype.Void) -def PyGILState_Release(space, ): - """Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding PyGILState_Ensure() call - (but generally this state will be unknown to the caller, hence the use of the - GILState API.) - - Every call to PyGILState_Ensure() must be matched by a call to - PyGILState_Release() on the same thread. - """ - raise NotImplementedError - @cpython_api([{int (*func)(void*}, {void*}], lltype.Void) def Py_AddPendingCall(space, , arg)): """ @@ -3579,12 +3543,6 @@ For values outside 0..LONG_MAX, both signed and unsigned integers are accepted.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyMapping_Check(space, o): - """Return 1 if the object provides mapping protocol, and 0 otherwise. This - function always succeeds.""" - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyMapping_Size(space, o): """ @@ -3631,13 +3589,6 @@ NULL. This is equivalent to the Python expression o.values().""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyMapping_Items(space, o): - """On success, return a list of the items in object o, where each item is a tuple - containing a key-value pair. On failure, return NULL. This is equivalent to - the Python expression o.items().""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyMapping_GetItemString(space, o, key): """Return element of o corresponding to the object key or NULL on failure. From arigo at codespeak.net Thu Apr 22 14:04:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 14:04:17 +0200 (CEST) Subject: [pypy-svn] r73967 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422120417.9F8DF282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 14:04:16 2010 New Revision: 73967 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Turn direct_calls into (for now) residual_call_xxx. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 22 14:04:16 2010 @@ -1,4 +1,6 @@ from pypy.rpython.lltypesystem import lltype +from pypy.jit.metainterp.history import getkind +from pypy.objspace.flow.model import SpaceOperation def transform_graph(graph): @@ -6,6 +8,8 @@ being flattened in a JitCode. """ for block in graph.iterblocks(): + for i in range(len(block.operations)): + block.operations[i] = rewrite_operation(block.operations[i]) optimize_goto_if_not(block) @@ -32,3 +36,47 @@ block.exitswitch = (op.opname,) + tuple(op.args) return True return False + +# ____________________________________________________________ + +def rewrite_operation(op): + try: + return _rewrite_ops[op.opname](op) + except KeyError: + return op + +def rewrite_op_direct_call(op): + """Turn direct_call(fn, i1, i2, ref1, ref2) + into residual_call_ir(fn, [i1, i2], [ref1, ref2]) + (or residual_call_r or residual_call_irf).""" + args_i = [] + args_r = [] + args_f = [] + for v in op.args[1:]: + add_in_correct_list(v, args_i, args_r, args_f) + if args_f: kinds = 'irf' + elif args_i: kinds = 'ir' + else: kinds = 'r' + sublists = [] + if 'i' in kinds: sublists.append(args_i) + if 'r' in kinds: sublists.append(args_r) + if 'f' in kinds: sublists.append(args_f) + return SpaceOperation('residual_call_' + kinds, + [op.args[0]] + sublists, + op.result) + +def add_in_correct_list(v, lst_i, lst_r, lst_f): + kind = getkind(v.concretetype) + if kind == 'void': return + elif kind == 'int': lst = lst_i + elif kind == 'ref': lst = lst_r + elif kind == 'float': lst = lst_f + else: raise AssertionError(kind) + lst.append(v) + +# ____________________________________________________________ + +_rewrite_ops = {} +for _name, _func in globals().items(): + if _name.startswith('rewrite_op_'): + _rewrite_ops[_name[len('rewrite_op_'):]] = _func Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu Apr 22 14:04:16 2010 @@ -1,7 +1,10 @@ +import random from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.jit.codewriter import jitter -from pypy.rpython.lltypesystem import lltype +from pypy.jit.metainterp.history import getkind +from pypy.rpython.lltypesystem import lltype, rclass, rstr +from pypy.translator.unsimplify import varoftype class FakeLink: args = [] @@ -47,3 +50,32 @@ block.exitswitch = v3 block.exits = [FakeLink(), FakeLink()] assert not jitter.optimize_goto_if_not(block) + +def test_residual_call(): + for with_i in [False, True]: + for with_r in [False, True]: + for with_f in [False, True]: + args = [] + if with_i: args += [lltype.Signed, lltype.Char] + if with_r: args += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] + if with_f: args += [lltype.Float, lltype.Float] + random.shuffle(args) + if with_f: expectedkind = 'irf' # all kinds + elif with_i: expectedkind = 'ir' # integers and references + else: expectedkind = 'r' # only references + yield residual_call_test, args, expectedkind + +def residual_call_test(argtypes, expectedkind): + FUNC = lltype.FuncType(argtypes, lltype.Signed) + fnptr = lltype.functionptr(FUNC, "g") # no graph + c_fnptr = Constant(fnptr, concretetype=lltype.typeOf(fnptr)) + vars = [varoftype(TYPE) for TYPE in argtypes] + op = SpaceOperation('direct_call', [c_fnptr] + vars, + varoftype(lltype.Signed)) + op1 = jitter.rewrite_operation(op) + assert op1.opname == 'residual_call_' + expectedkind + assert op1.args[0] == c_fnptr + assert len(op1.args) == 1 + len(expectedkind) + for sublist, kind in zip(op1.args[1:], expectedkind): + assert sublist == [v for v in vars + if getkind(v.concretetype).startswith(kind)] From afa at codespeak.net Thu Apr 22 14:07:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 14:07:08 +0200 (CEST) Subject: [pypy-svn] r73968 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422120708.165B8282B9E@codespeak.net> Author: afa Date: Thu Apr 22 14:07:06 2010 New Revision: 73968 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: PyString_AsStringAndSize Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Thu Apr 22 14:07:06 2010 @@ -84,6 +84,29 @@ ref_str.c_buffer = rffi.str2charp(s) return ref_str.c_buffer + at cpython_api([PyObject, rffi.CCHARPP, rffi.CArrayPtr(Py_ssize_t)], rffi.INT_real, error=-1) +def PyString_AsStringAndSize(space, ref, buffer, length): + if not PyString_Check(space, ref): + raise OperationError(space.w_TypeError, space.wrap( + "PyString_AsStringAndSize only support strings")) + ref_str = rffi.cast(PyStringObject, ref) + if not ref_str.c_buffer: + # copy string buffer + w_str = from_ref(space, ref) + s = space.str_w(w_str) + ref_str.c_buffer = rffi.str2charp(s) + buffer[0] = ref_str.c_buffer + if length: + length[0] = ref_str.c_size + else: + i = 0 + while ref_str.c_buffer[i] != '\0': + i += 1 + if i != ref_str.c_size: + raise OperationError(space.w_TypeError, space.wrap( + "expected string without null bytes")) + return 0 + @cpython_api([PyObject], Py_ssize_t, error=-1) def PyString_Size(space, ref): if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Thu Apr 22 14:07:06 2010 @@ -105,6 +105,31 @@ )]) assert module.string_as_string("huheduwe") == "huhe" + def test_AsStringAndSize(self): + module = self.import_extension('foo', [ + ("getstring", "METH_NOARGS", + """ + PyObject* s1 = PyString_FromStringAndSize("te\\0st", 5); + char *buf; + Py_ssize_t len; + if (PyString_AsStringAndSize(s1, &buf, &len) < 0) + return NULL; + if (len != 5) { + PyErr_SetString(PyExc_AssertionError, "Bad Length"); + return NULL; + } + if (PyString_AsStringAndSize(s1, &buf, NULL) >= 0) { + PyErr_SetString(PyExc_AssertionError, "Should Have failed"); + return NULL; + } + PyErr_Clear(); + Py_DECREF(s1); + Py_INCREF(Py_None); + return Py_None; + """), + ]) + module.getstring() + def test_format_v(self): module = self.import_extension('foo', [ ("test_string_format_v", "METH_VARARGS", From arigo at codespeak.net Thu Apr 22 14:10:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 14:10:56 +0200 (CEST) Subject: [pypy-svn] r73970 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422121056.B863E282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 14:10:54 2010 New Revision: 73970 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Also encode the result type in the operation name. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 22 14:10:54 2010 @@ -46,9 +46,9 @@ return op def rewrite_op_direct_call(op): - """Turn direct_call(fn, i1, i2, ref1, ref2) - into residual_call_ir(fn, [i1, i2], [ref1, ref2]) - (or residual_call_r or residual_call_irf).""" + """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' + into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. + The name is one of 'residual_call_{r,ir,irf}_{i,r,f,v}'.""" args_i = [] args_r = [] args_f = [] @@ -61,7 +61,8 @@ if 'i' in kinds: sublists.append(args_i) if 'r' in kinds: sublists.append(args_r) if 'f' in kinds: sublists.append(args_f) - return SpaceOperation('residual_call_' + kinds, + reskind = getkind(op.result.concretetype)[0] + return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), [op.args[0]] + sublists, op.result) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu Apr 22 14:10:54 2010 @@ -52,28 +52,32 @@ assert not jitter.optimize_goto_if_not(block) def test_residual_call(): - for with_i in [False, True]: - for with_r in [False, True]: - for with_f in [False, True]: - args = [] - if with_i: args += [lltype.Signed, lltype.Char] - if with_r: args += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] - if with_f: args += [lltype.Float, lltype.Float] - random.shuffle(args) - if with_f: expectedkind = 'irf' # all kinds - elif with_i: expectedkind = 'ir' # integers and references - else: expectedkind = 'r' # only references - yield residual_call_test, args, expectedkind + for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, + lltype.Float, lltype.Void]: + for with_i in [False, True]: + for with_r in [False, True]: + for with_f in [False, True]: + ARGS = [] + if with_i: ARGS += [lltype.Signed, lltype.Char] + if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] + if with_f: ARGS += [lltype.Float, lltype.Float] + random.shuffle(ARGS) + if with_f: expectedkind = 'irf' # all kinds + elif with_i: expectedkind = 'ir' # integers and references + else: expectedkind = 'r' # only references + yield residual_call_test, ARGS, RESTYPE, expectedkind -def residual_call_test(argtypes, expectedkind): - FUNC = lltype.FuncType(argtypes, lltype.Signed) +def residual_call_test(argtypes, restype, expectedkind): + FUNC = lltype.FuncType(argtypes, restype) fnptr = lltype.functionptr(FUNC, "g") # no graph c_fnptr = Constant(fnptr, concretetype=lltype.typeOf(fnptr)) vars = [varoftype(TYPE) for TYPE in argtypes] - op = SpaceOperation('direct_call', [c_fnptr] + vars, - varoftype(lltype.Signed)) + v_result = varoftype(restype) + op = SpaceOperation('direct_call', [c_fnptr] + vars, v_result) op1 = jitter.rewrite_operation(op) - assert op1.opname == 'residual_call_' + expectedkind + reskind = getkind(restype)[0] + assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) + assert op1.result == v_result assert op1.args[0] == c_fnptr assert len(op1.args) == 1 + len(expectedkind) for sublist, kind in zip(op1.args[1:], expectedkind): From afa at codespeak.net Thu Apr 22 14:16:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 14:16:20 +0200 (CEST) Subject: [pypy-svn] r73971 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422121620.918B9282B9E@codespeak.net> Author: afa Date: Thu Apr 22 14:16:18 2010 New Revision: 73971 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Log: Py_AsUnsignedLong Modified: pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/intobject.py Thu Apr 22 14:16:18 2010 @@ -21,6 +21,13 @@ there was an error, or whether the value just happened to be -1.""" return space.int_w(space.int(w_obj)) + at cpython_api([PyObject], lltype.Unsigned, error=-1) +def PyInt_AsUnsignedLong(space, w_obj): + """Return a C unsigned long representation of the contents of pylong. + If pylong is greater than ULONG_MAX, an OverflowError is + raised.""" + return space.uint_w(space.int(w_obj)) + @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) def PyInt_AS_LONG(space, w_int): """Return the value of the object w_int. No error checking is performed.""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_intobject.py Thu Apr 22 14:16:18 2010 @@ -19,6 +19,11 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint + assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 + assert api.PyErr_Occurred() is space.w_ValueError + api.PyErr_Clear() + def test_coerce(self, space, api): class Coerce(object): def __int__(self): From afa at codespeak.net Thu Apr 22 14:17:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 14:17:48 +0200 (CEST) Subject: [pypy-svn] r73972 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100422121748.17693282B9E@codespeak.net> Author: afa Date: Thu Apr 22 14:17:46 2010 New Revision: 73972 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Log: Implement more basic types for struct members Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Thu Apr 22 14:17:46 2010 @@ -21,10 +21,15 @@ /* Types */ +#define T_SHORT 0 #define T_INT 1 +#define T_LONG 2 #define T_STRING 5 #define T_OBJECT 6 #define T_CHAR 7 /* 1-character string */ +#define T_USHORT 10 +#define T_UINT 11 +#define T_ULONG 12 #define T_STRING_INPLACE 13 /* Strings contained in the structure */ #define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError when the value is NULL, instead of Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Thu Apr 22 14:17:46 2010 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext import structmemberdefs from pypy.module.cpyext.api import ADDR, PyObjectP, cpython_api -from pypy.module.cpyext.intobject import PyInt_AsLong +from pypy.module.cpyext.intobject import PyInt_AsLong, PyInt_AsUnsignedLong from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, from_ref, make_ref from pypy.module.cpyext.stringobject import (PyString_FromString, @@ -16,9 +16,24 @@ addr = rffi.cast(ADDR, obj) addr += w_member.c_offset member_type = rffi.cast(lltype.Signed, w_member.c_type) - if member_type == structmemberdefs.T_INT: + if member_type == structmemberdefs.T_SHORT: + result = rffi.cast(rffi.SHORTP, addr) + w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_INT: result = rffi.cast(rffi.INTP, addr) w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_LONG: + result = rffi.cast(rffi.LONGP, addr) + w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_USHORT: + result = rffi.cast(rffi.USHORTP, addr) + w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_UINT: + result = rffi.cast(rffi.UINTP, addr) + w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_ULONG: + result = rffi.cast(rffi.ULONGP, addr) + w_result = space.wrap(result[0]) elif member_type == structmemberdefs.T_STRING: result = rffi.cast(rffi.CCHARPP, addr) if result[0]: @@ -68,10 +83,30 @@ raise OperationError(space.w_TypeError, space.wrap("can't delete numeric/char attribute")) - if member_type == structmemberdefs.T_INT: + if member_type == structmemberdefs.T_SHORT: + w_long_value = PyInt_AsLong(space, w_value) + array = rffi.cast(rffi.SHORTP, addr) + array[0] = rffi.cast(rffi.SHORT, w_long_value) + elif member_type == structmemberdefs.T_INT: w_long_value = PyInt_AsLong(space, w_value) array = rffi.cast(rffi.INTP, addr) array[0] = rffi.cast(rffi.INT, w_long_value) + elif member_type == structmemberdefs.T_LONG: + w_long_value = PyInt_AsLong(space, w_value) + array = rffi.cast(rffi.LONGP, addr) + array[0] = rffi.cast(rffi.LONG, w_long_value) + elif member_type == structmemberdefs.T_USHORT: + w_long_value = PyInt_AsUnsignedLong(space, w_value) + array = rffi.cast(rffi.USHORTP, addr) + array[0] = rffi.cast(rffi.USHORT, w_long_value) + elif member_type == structmemberdefs.T_UINT: + w_long_value = PyInt_AsUnsignedLong(space, w_value) + array = rffi.cast(rffi.UINTP, addr) + array[0] = rffi.cast(rffi.UINT, w_long_value) + elif member_type == structmemberdefs.T_ULONG: + w_long_value = PyInt_AsUnsignedLong(space, w_value) + array = rffi.cast(rffi.ULONGP, addr) + array[0] = rffi.cast(rffi.ULONG, w_long_value) elif member_type == structmemberdefs.T_CHAR: str_value = space.str_w(w_value) if len(str_value) != 1: Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Thu Apr 22 14:17:46 2010 @@ -1,7 +1,12 @@ +T_SHORT = 0 T_INT = 1 +T_LONG = 2 T_STRING = 5 T_OBJECT = 6 T_CHAR = 7 +T_USHORT = 10 +T_UINT = 11 +T_ULONG = 12 T_STRING_INPLACE = 13 T_OBJECT_EX = 16 From arigo at codespeak.net Thu Apr 22 14:39:06 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 14:39:06 +0200 (CEST) Subject: [pypy-svn] r73973 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422123906.18308282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 14:39:04 2010 New Revision: 73973 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Support for lists of variables in the assembler source code. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu Apr 22 14:39:04 2010 @@ -39,7 +39,7 @@ flattener = GraphFlattener(graph, regallocs) flattener.enforce_input_args() flattener.generate_ssa_form() - return flattener.assembler + return flattener.ssarepr class GraphFlattener(object): @@ -48,6 +48,11 @@ self.graph = graph self.regallocs = regallocs self.registers = {} + if graph: + name = graph.name + else: + name = '?' + self.ssarepr = SSARepr(name) def enforce_input_args(self): inputargs = self.graph.startblock.inputargs @@ -64,7 +69,6 @@ self.regallocs[kind].swapcolors(realcol, curcol) def generate_ssa_form(self): - self.assembler = SSARepr(self.graph.name) self.seen_blocks = {} self.make_bytecode_block(self.graph.startblock) @@ -160,13 +164,20 @@ self.emitline('%s_rename' % kind, frm, to) def emitline(self, *line): - self.assembler.insns.append(line) + self.ssarepr.insns.append(line) - def flatten_list(self, list): + def flatten_list(self, arglist): args = [] - for v in list: + for v in arglist: if isinstance(v, Variable): v = self.getcolor(v) + elif isinstance(v, list): + lst = v + v = [] + for x in lst: + if isinstance(x, Variable): + x = self.getcolor(x) + v.append(x) args.append(v) return args Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Thu Apr 22 14:39:04 2010 @@ -18,7 +18,7 @@ elif isinstance(x, list): return '[%s]' % ', '.join(map(repr, x)) else: - return `x` # ? + return '' % (x,) # seenlabels = {} for asm in ssarepr.insns: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Thu Apr 22 14:39:04 2010 @@ -1,8 +1,12 @@ import py from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list +from pypy.jit.codewriter.flatten import GraphFlattener from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitter import transform_graph +from pypy.rpython.lltypesystem import lltype, rclass, rstr +from pypy.objspace.flow.model import SpaceOperation, Variable, Constant +from pypy.translator.unsimplify import varoftype class FakeRegAlloc: @@ -16,6 +20,11 @@ self.num_colors += 1 return self.seen[v] +def fake_regallocs(): + return {'int': FakeRegAlloc(), + 'ref': FakeRegAlloc(), + 'float': FakeRegAlloc()} + def test_reorder_renaming_list(): result = reorder_renaming_list([], []) assert result == [] @@ -35,13 +44,14 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def encoding_test(self, func, args, expected, optimize=False): + def encoding_test(self, func, args, expected, transform=False): graphs = self.make_graphs(func, args) - if optimize: + if transform: transform_graph(graphs[0]) - ssarepr = flatten_graph(graphs[0], {'int': FakeRegAlloc(), - 'ref': FakeRegAlloc(), - 'float': FakeRegAlloc()}) + ssarepr = flatten_graph(graphs[0], fake_regallocs()) + self.assert_format(ssarepr, expected) + + def assert_format(self, ssarepr, expected): asm = format_assembler(ssarepr) expected = str(py.code.Source(expected)).strip() + '\n' assert asm == expected @@ -97,7 +107,7 @@ goto L1 L2: int_return %i3 - """, optimize=True) + """, transform=True) def test_float(self): def f(i, f): @@ -109,3 +119,20 @@ float_add %f2, %f1, %f3 float_return %f3 """) + + def test_arg_sublist_1(self): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Char) + v3 = varoftype(rclass.OBJECTPTR) + v4 = varoftype(lltype.Ptr(rstr.STR)) + v5 = varoftype(lltype.Float) + op = SpaceOperation('residual_call_ir_f', + [Constant(12345, lltype.Signed), # function ptr + [v1, v2], # int args + [v3, v4]], # ref args + v5) # result + flattener = GraphFlattener(None, fake_regallocs()) + flattener.serialize_op(op) + self.assert_format(flattener.ssarepr, """ + residual_call_ir_f $12345, [%i0, %i1], [%r0, %r1], %f0 + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu Apr 22 14:39:04 2010 @@ -67,19 +67,23 @@ else: expectedkind = 'r' # only references yield residual_call_test, ARGS, RESTYPE, expectedkind -def residual_call_test(argtypes, restype, expectedkind): +def get_direct_call_op(argtypes, restype): FUNC = lltype.FuncType(argtypes, restype) fnptr = lltype.functionptr(FUNC, "g") # no graph c_fnptr = Constant(fnptr, concretetype=lltype.typeOf(fnptr)) vars = [varoftype(TYPE) for TYPE in argtypes] v_result = varoftype(restype) op = SpaceOperation('direct_call', [c_fnptr] + vars, v_result) + return op + +def residual_call_test(argtypes, restype, expectedkind): + op = get_direct_call_op(argtypes, restype) op1 = jitter.rewrite_operation(op) reskind = getkind(restype)[0] assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) - assert op1.result == v_result - assert op1.args[0] == c_fnptr + assert op1.result == op.result + assert op1.args[0] == op.args[0] assert len(op1.args) == 1 + len(expectedkind) for sublist, kind in zip(op1.args[1:], expectedkind): - assert sublist == [v for v in vars + assert sublist == [v for v in op.args[1:] if getkind(v.concretetype).startswith(kind)] From arigo at codespeak.net Thu Apr 22 15:19:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 15:19:15 +0200 (CEST) Subject: [pypy-svn] r73975 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422131915.69CFC282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 15:19:13 2010 New Revision: 73975 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Proper support for lists of registers. They are actually ListOfKinds, not just lists. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Thu Apr 22 15:19:13 2010 @@ -1,5 +1,6 @@ from pypy.jit.metainterp.history import AbstractValue, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS +from pypy.jit.codewriter.flatten import ListOfKind from pypy.objspace.flow.model import Constant @@ -30,72 +31,109 @@ self.insns = {} def assemble(self, ssarepr): - code = [] - constants_dict = {} - constants_i = [] - constants_r = [] - constants_f = [] - label_positions = {} - tlabel_positions = [] - highest_regs = dict.fromkeys(KINDS, 0) + self.setup() for insn in ssarepr.insns: - if isinstance(insn[0], Label): - label_positions[insn[0].name] = len(code) - continue - startposition = len(code) - code.append("temporary placeholder") - # - argcodes = [] - for x in insn[1:]: - if isinstance(x, Register): - if x.index > highest_regs[x.kind]: - highest_regs[x.kind] = x.index - code.append(chr(x.index)) - argcodes.append(x.kind[0]) - elif isinstance(x, Constant): - kind = getkind(x.concretetype) - if kind == 'int' and -128 <= x.value <= 127: - code.append(chr(x.value & 0xFF)) - argcodes.append('c') - else: - if x not in constants_dict: - if kind == 'int': - constants = constants_i - elif kind == 'ref': - constants = constants_r - elif kind == 'float': - constants = constants_f - else: - raise NotImplementedError(x) - constants.append(x.value) - constants_dict[x] = 256 - len(constants) - code.append(chr(constants_dict[x])) - argcodes.append(kind[0]) - elif isinstance(x, TLabel): - tlabel_positions.append((x.name, len(code))) - code.append("temp 1") - code.append("temp 2") - argcodes.append('L') + self.write_insn(insn) + self.fix_labels() + self.check_result() + return self.make_jitcode(ssarepr.name) + + def setup(self): + self.code = [] + self.constants_dict = {} + self.constants_i = [] + self.constants_r = [] + self.constants_f = [] + self.label_positions = {} + self.tlabel_positions = [] + self.highest_regs = dict.fromkeys(KINDS, 0) + + def emit_reg(self, reg): + if reg.index > self.highest_regs[reg.kind]: + self.highest_regs[reg.kind] = reg.index + self.code.append(chr(reg.index)) + + def emit_const(self, const, kind): + if const not in self.constants_dict: + if kind == 'int': + constants = self.constants_i + elif kind == 'ref': + constants = self.constants_r + elif kind == 'float': + constants = self.constants_f + else: + raise NotImplementedError(const) + constants.append(const.value) + self.constants_dict[const] = 256 - len(constants) + self.code.append(chr(self.constants_dict[const])) + + def write_insn(self, insn): + if isinstance(insn[0], Label): + self.label_positions[insn[0].name] = len(self.code) + return + startposition = len(self.code) + self.code.append("temporary placeholder") + # + argcodes = [] + for x in insn[1:]: + if isinstance(x, Register): + self.emit_reg(x) + argcodes.append(x.kind[0]) + elif isinstance(x, Constant): + kind = getkind(x.concretetype) + if kind == 'int' and -128 <= x.value <= 127: + self.code.append(chr(x.value & 0xFF)) + argcodes.append('c') else: - raise NotImplementedError(x) - # - key = insn[0] + '/' + ''.join(argcodes) - num = self.insns.setdefault(key, len(self.insns)) - code[startposition] = chr(num) + self.emit_const(x, kind) + argcodes.append(kind[0]) + elif isinstance(x, TLabel): + self.tlabel_positions.append((x.name, len(self.code))) + self.code.append("temp 1") + self.code.append("temp 2") + argcodes.append('L') + elif isinstance(x, ListOfKind): + itemkind = x.kind + lst = list(x) + assert len(lst) <= 255, "list too long!" + self.code.append(chr(len(lst))) + for item in lst: + if isinstance(item, Register): + assert itemkind == item.kind + self.emit_reg(item) + elif isinstance(item, Constant): + assert itemkind == getkind(item.concretetype) + self.emit_const(item, itemkind) + else: + raise NotImplementedError("found in ListOfKind(): %r" + % (item,)) + argcodes.append(itemkind[0].upper()) + else: + raise NotImplementedError(x) # - for name, pos in tlabel_positions: - assert code[pos ] == "temp 1" - assert code[pos+1] == "temp 2" - target = label_positions[name] + key = insn[0] + '/' + ''.join(argcodes) + num = self.insns.setdefault(key, len(self.insns)) + self.code[startposition] = chr(num) + + def fix_labels(self): + for name, pos in self.tlabel_positions: + assert self.code[pos ] == "temp 1" + assert self.code[pos+1] == "temp 2" + target = self.label_positions[name] assert 0 <= target <= 0xFFFF - code[pos ] = chr(target & 0xFF) - code[pos+1] = chr(target >> 8) - # + self.code[pos ] = chr(target & 0xFF) + self.code[pos+1] = chr(target >> 8) + + def check_result(self): # Limitation of the number of registers, from the single-byte encoding - assert highest_regs['int'] + len(constants_i) <= 256 - assert highest_regs['ref'] + len(constants_r) <= 256 - assert highest_regs['float'] + len(constants_f) <= 256 - # - jitcode = JitCode(ssarepr.name) - jitcode.setup(''.join(code), constants_i, constants_r, constants_f) + assert self.highest_regs['int'] + len(self.constants_i) <= 256 + assert self.highest_regs['ref'] + len(self.constants_r) <= 256 + assert self.highest_regs['float'] + len(self.constants_f) <= 256 + + def make_jitcode(self, name): + jitcode = JitCode(name) + jitcode.setup(''.join(self.code), + self.constants_i, + self.constants_r, + self.constants_f) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu Apr 22 15:19:13 2010 @@ -1,4 +1,4 @@ -from pypy.objspace.flow.model import Variable +from pypy.objspace.flow.model import Variable, Constant from pypy.jit.metainterp.history import getkind from pypy.rpython.lltypesystem import lltype @@ -26,9 +26,22 @@ class Register(object): def __init__(self, kind, index): - self.kind = kind + self.kind = kind # 'int', 'ref' or 'float' self.index = index +class ListOfKind(object): + # a list of Regs/Consts, all of the same 'kind'. + # We cannot use a plain list, because we wouldn't know what 'kind' of + # Regs/Consts would be expected in case the list is empty. + def __init__(self, kind, content): + assert kind in KINDS + self.kind = kind + self.content = tuple(content) + def __repr__(self): + return '%s%s' % (self.kind[0], self.content) + def __iter__(self): + return iter(self.content) + KINDS = ['int', 'ref', 'float'] # ____________________________________________________________ @@ -161,6 +174,8 @@ for v, w in result: self.emitline('%s_copy' % kind, v, w) else: + frm = ListOfKind(kind, frm) + to = ListOfKind(kind, to) self.emitline('%s_rename' % kind, frm, to) def emitline(self, *line): @@ -171,13 +186,17 @@ for v in arglist: if isinstance(v, Variable): v = self.getcolor(v) - elif isinstance(v, list): - lst = v - v = [] - for x in lst: + elif isinstance(v, Constant): + pass + elif isinstance(v, ListOfKind): + lst = [] + for x in v: if isinstance(x, Variable): x = self.getcolor(x) - v.append(x) + lst.append(x) + v = ListOfKind(v.kind, lst) + else: + raise NotImplementedError(type(v)) args.append(v) return args Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Thu Apr 22 15:19:13 2010 @@ -1,9 +1,10 @@ import py from pypy.objspace.flow.model import Constant from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register +from pypy.jit.codewriter.flatten import ListOfKind -def format_assembler(ssarepr): +def format_assembler(ssarepr, dump=True): """For testing: format a SSARepr as a multiline string.""" from cStringIO import StringIO seen = {} @@ -15,8 +16,8 @@ return '$' + str(x.value) elif isinstance(x, TLabel): return getlabelname(x) - elif isinstance(x, list): - return '[%s]' % ', '.join(map(repr, x)) + elif isinstance(x, ListOfKind): + return '%s[%s]' % (x.kind[0], ', '.join(map(repr, x))) else: return '' % (x,) # @@ -43,4 +44,8 @@ print >> output, ', '.join(map(repr, asm[1:])) else: print >> output - return output.getvalue() + res = output.getvalue() + if dump: + print res + return res + Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 22 15:19:13 2010 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation +from pypy.jit.codewriter.flatten import ListOfKind def transform_graph(graph): @@ -58,9 +59,9 @@ elif args_i: kinds = 'ir' else: kinds = 'r' sublists = [] - if 'i' in kinds: sublists.append(args_i) - if 'r' in kinds: sublists.append(args_r) - if 'f' in kinds: sublists.append(args_f) + if 'i' in kinds: sublists.append(ListOfKind('int', args_i)) + if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) + if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) reskind = getkind(op.result.concretetype)[0] return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), [op.args[0]] + sublists, Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Thu Apr 22 15:19:13 2010 @@ -1,5 +1,6 @@ from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register +from pypy.jit.codewriter.flatten import ListOfKind from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype @@ -79,3 +80,16 @@ 'int_sub/ici': 2, 'goto/L': 3, 'int_return/i': 4} + +def test_assemble_list(): + ssarepr = SSARepr("test") + i0, i1 = Register('int', 0x16), Register('int', 0x17) + ssarepr.insns = [ + ('foobar', ListOfKind('int', [i0, i1, Constant(42, lltype.Signed)]), + ListOfKind('ref', [])), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == "\x00\x03\x16\x17\xFF\x00" + assert assembler.insns == {'foobar/IR': 0} + assert jitcode.constants_i == [42] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Thu Apr 22 15:19:13 2010 @@ -1,7 +1,7 @@ import py from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list -from pypy.jit.codewriter.flatten import GraphFlattener +from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitter import transform_graph from pypy.rpython.lltypesystem import lltype, rclass, rstr @@ -128,11 +128,11 @@ v5 = varoftype(lltype.Float) op = SpaceOperation('residual_call_ir_f', [Constant(12345, lltype.Signed), # function ptr - [v1, v2], # int args - [v3, v4]], # ref args + ListOfKind('int', [v1, v2]), # int args + ListOfKind('ref', [v3, v4])], # ref args v5) # result flattener = GraphFlattener(None, fake_regallocs()) flattener.serialize_op(op) self.assert_format(flattener.ssarepr, """ - residual_call_ir_f $12345, [%i0, %i1], [%r0, %r1], %f0 + residual_call_ir_f $12345, i[%i0, %i1], r[%r0, %r1], %f0 """) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Thu Apr 22 15:19:13 2010 @@ -2,19 +2,19 @@ from pypy.objspace.flow.model import Constant from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.flatten import Label, TLabel, SSARepr, Register +from pypy.jit.codewriter.flatten import ListOfKind +from pypy.rpython.lltypesystem import lltype def test_format_assembler_simple(): ssarepr = SSARepr("test") i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) ssarepr.insns = [ - ('foobar', [i0, i1]), ('int_add', i0, i1, i2), ('int_return', i2), ] asm = format_assembler(ssarepr) expected = """ - foobar [%i0, %i1] int_add %i0, %i1, %i2 int_return %i2 """ @@ -36,7 +36,6 @@ ssarepr = SSARepr("test") i0, i1 = Register('int', 0), Register('int', 1) ssarepr.insns = [ - ('foobar', [i0, i1]), (Label('L1'),), ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0)), ('int_add', i1, i0, i1), @@ -47,7 +46,6 @@ ] asm = format_assembler(ssarepr) expected = """ - foobar [%i0, %i1] L1: goto_if_not_int_gt L2, %i0, $0 int_add %i1, %i0, %i1 @@ -57,3 +55,15 @@ int_return %i1 """ assert asm == str(py.code.Source(expected)).strip() + '\n' + +def test_format_assembler_list(): + ssarepr = SSARepr("test") + i0, i1 = Register('int', 0), Register('int', 1) + ssarepr.insns = [ + ('foobar', ListOfKind('int', [i0, Constant(123, lltype.Signed), i1])), + ] + asm = format_assembler(ssarepr) + expected = """ + foobar i[%i0, $123, %i1] + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu Apr 22 15:19:13 2010 @@ -84,6 +84,7 @@ assert op1.result == op.result assert op1.args[0] == op.args[0] assert len(op1.args) == 1 + len(expectedkind) - for sublist, kind in zip(op1.args[1:], expectedkind): - assert sublist == [v for v in op.args[1:] - if getkind(v.concretetype).startswith(kind)] + for sublist, kind1 in zip(op1.args[1:], expectedkind): + assert sublist.kind.startswith(kind1) + assert list(sublist) == [v for v in op.args[1:] + if getkind(v.concretetype) == sublist.kind] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Thu Apr 22 15:19:13 2010 @@ -69,7 +69,7 @@ L1: int_gt %i0, $0, %i2 goto_if_not L2, %i2 - int_rename [%i1, %i0], [%i0, %i1] + int_rename i[%i1, %i0], i[%i0, %i1] goto L1 L2: int_return %i1 @@ -102,7 +102,7 @@ L1: int_gt %i0, $0, %i3 goto_if_not L2, %i3 - int_rename [%i1, %i2, %i0], [%i0, %i1, %i2] + int_rename i[%i1, %i2, %i0], i[%i0, %i1, %i2] goto L1 L2: int_return %i1 From arigo at codespeak.net Thu Apr 22 15:57:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 15:57:09 +0200 (CEST) Subject: [pypy-svn] r73976 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100422135709.6E5CE282B9E@codespeak.net> Author: arigo Date: Thu Apr 22 15:57:07 2010 New Revision: 73976 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Start to write one of the residual_call instructions. Very incomplete. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 22 15:57:07 2010 @@ -91,10 +91,23 @@ next_argcode = next_argcode + 1 value = ord(code[position]) | (ord(code[position+1])<<8) position += 2 + elif argtype == 'I' or argtype == 'R' or argtype == 'F': + assert argcodes[next_argcode] == argtype + next_argcode = next_argcode + 1 + length = ord(code[position]) + position += 1 + value = [] + for i in range(length): + index = ord(code[position+i]) + if argtype == 'I': reg = self.registers_i[index] + elif argtype == 'R': reg = self.registers_r[index] + elif argtype == 'F': reg = self.registers_f[index] + value.append(reg) + position += length elif argtype == 'pc': value = position else: - raise AssertionError("bad argtype") + raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) result = boundmethod(*args) if resulttype == 'i': @@ -131,6 +144,7 @@ pass # XXX must be specialized + # XXX the real performance impact of the following loop is unclear def copy_constants(self, registers, constants): """Copy jitcode.constants[0] to registers[255], jitcode.constants[1] to registers[254], @@ -184,3 +198,8 @@ @arguments("L", returns="L") def opimpl_goto(self, target): return target + + @arguments("i", "I", "R", returns="i") + def opimpl_residual_call_ir_i(self, function, args_i, args_r): + # XXX! + return function(*args_i) From arigo at codespeak.net Thu Apr 22 16:42:26 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 16:42:26 +0200 (CEST) Subject: [pypy-svn] r73977 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422144226.A12EE282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 16:42:25 2010 New Revision: 73977 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Mostly no-op. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Thu Apr 22 16:42:25 2010 @@ -58,16 +58,15 @@ pass die_index += 1 if getkind(op.result.concretetype) == self.kind: - livevars.add(op.result) dg.add_node(op.result) for v in livevars: if getkind(v.concretetype) == self.kind: dg.add_edge(v, op.result) + livevars.add(op.result) self._depgraph = dg def coalesce_variables(self): - uf = UnionFind() - dg = self._depgraph + self._unionfind = UnionFind() pendingblocks = list(self.graph.iterblocks()) while pendingblocks: block = pendingblocks.pop() @@ -79,19 +78,22 @@ # middle during blackholing. for link in block.exits: for i, v in enumerate(link.args): - if (isinstance(v, Variable) and - getkind(v.concretetype) == self.kind): - w = link.target.inputargs[i] - v0 = uf.find_rep(v) - w0 = uf.find_rep(w) - if v0 is not w0 and v0 not in dg.neighbours[w0]: - _, rep, _ = uf.union(v0, w0) - if rep is v0: - dg.coalesce(w0, v0) - else: - assert rep is w0 - dg.coalesce(v0, w0) - self._unionfind = uf + self._try_coalesce(v, link.target.inputargs[i]) + + def _try_coalesce(self, v, w): + if isinstance(v, Variable) and getkind(v.concretetype) == self.kind: + dg = self._depgraph + uf = self._unionfind + v0 = uf.find_rep(v) + w0 = uf.find_rep(w) + if v0 is not w0 and v0 not in dg.neighbours[w0]: + _, rep, _ = uf.union(v0, w0) + assert uf.find_rep(v0) is uf.find_rep(w0) is rep + if rep is v0: + dg.coalesce(w0, v0) + else: + assert rep is w0 + dg.coalesce(v0, w0) def find_node_coloring(self): self._coloring = self._depgraph.find_node_coloring() Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Thu Apr 22 16:42:25 2010 @@ -8,7 +8,8 @@ class TestRegAlloc: def make_graphs(self, func, values, type_system='lltype'): - self.rtyper = support.annotate(func, values, type_system=type_system) + self.rtyper = support.annotate(func, values, type_system=type_system, + backendoptimize=False) return self.rtyper.annotator.translator.graphs def check_assembler(self, graph, expected): From arigo at codespeak.net Thu Apr 22 16:43:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 16:43:00 +0200 (CEST) Subject: [pypy-svn] r73978 - pypy/branch/blackhole-improvement/pypy/tool/algo Message-ID: <20100422144300.65A02282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 16:42:58 2010 New Revision: 73978 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Log: Add asserts. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Thu Apr 22 16:42:58 2010 @@ -11,6 +11,7 @@ self.neighbours[v] = set() def add_edge(self, v1, v2): + assert v1 != v2 self.neighbours[v1].add(v2) self.neighbours[v2].add(v1) @@ -18,6 +19,7 @@ """Remove vold from the graph, and attach all its edges to vnew.""" for n in self.neighbours.pop(vold): self.neighbours[n].remove(vold) + assert vnew != n self.neighbours[n].add(vnew) self.neighbours[vnew].add(n) # we should remove vold from self._all_nodes, but it's too costly From arigo at codespeak.net Thu Apr 22 16:58:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 16:58:27 +0200 (CEST) Subject: [pypy-svn] r73979 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100422145827.EA458282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 16:58:26 2010 New Revision: 73979 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Remove same_as and other operations that are no-ops at this level. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu Apr 22 16:58:26 2010 @@ -153,14 +153,12 @@ def insert_renamings(self, link): renamings = {} - lst = [(v, self.getcolor(link.target.inputargs[i])) + lst = [(self.getcolor(v), self.getcolor(link.target.inputargs[i])) for i, v in enumerate(link.args)] lst.sort(key=lambda(v, w): w.index) for v, w in lst: - if isinstance(v, Variable): - v = self.getcolor(v) - if v == w: - continue + if v == w: + continue frm, to = renamings.setdefault(w.kind, ([], [])) frm.append(v) to.append(w) @@ -189,11 +187,7 @@ elif isinstance(v, Constant): pass elif isinstance(v, ListOfKind): - lst = [] - for x in v: - if isinstance(x, Variable): - x = self.getcolor(x) - lst.append(x) + lst = [self.getcolor(x) for x in v] v = ListOfKind(v.kind, lst) else: raise NotImplementedError(type(v)) @@ -207,6 +201,8 @@ self.emitline(op.opname, *args) def getcolor(self, v): + if isinstance(v, Constant): + return v kind = getkind(v.concretetype) col = self.regallocs[kind].getcolor(v) # if kind=='void', fix caller try: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 22 16:58:26 2010 @@ -9,8 +9,18 @@ being flattened in a JitCode. """ for block in graph.iterblocks(): - for i in range(len(block.operations)): - block.operations[i] = rewrite_operation(block.operations[i]) + rename = {} + newoperations = [] + for op in block.operations: + if is_noop_operation(op): + rename[op.result] = rename.get(op.args[0], op.args[0]) + else: + for i, v in enumerate(op.args): + if v in rename: + op = SpaceOperation(op.opname, op.args[:], op.result) + op.args[i] = rename[v] + newoperations.append(rewrite_operation(op)) + block.operations = newoperations optimize_goto_if_not(block) @@ -40,6 +50,13 @@ # ____________________________________________________________ +def is_noop_operation(op): + return op.opname in _noop_operations + +_noop_operations = {'same_as': True, + 'cast_int_to_char': True, + 'cast_char_to_int': True} + def rewrite_operation(op): try: return _rewrite_ops[op.opname](op) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Thu Apr 22 16:58:26 2010 @@ -136,3 +136,12 @@ self.assert_format(flattener.ssarepr, """ residual_call_ir_f $12345, i[%i0, %i1], r[%r0, %r1], %f0 """) + + def test_same_as_removal(self): + def f(a): + b = chr(a) + return ord(b) + a + self.encoding_test(f, [65], """ + int_add %i0, %i0, %i1 + int_return %i1 + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Thu Apr 22 16:58:26 2010 @@ -8,8 +8,7 @@ class TestRegAlloc: def make_graphs(self, func, values, type_system='lltype'): - self.rtyper = support.annotate(func, values, type_system=type_system, - backendoptimize=False) + self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs def check_assembler(self, graph, expected): From arigo at codespeak.net Thu Apr 22 17:22:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 22 Apr 2010 17:22:24 +0200 (CEST) Subject: [pypy-svn] r73980 - in pypy/branch/blackhole-improvement/pypy/jit/metainterp: . test Message-ID: <20100422152224.E24A5282BAD@codespeak.net> Author: arigo Date: Thu Apr 22 17:22:23 2010 New Revision: 73980 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Add some more cases. Lots of copy-pasting... Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 22 17:22:23 2010 @@ -189,12 +189,47 @@ return a @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_lt(self, target, a, b, pc): + if a < b: + return pc + else: + return target + + @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_le(self, target, a, b, pc): + if a <= b: + return pc + else: + return target + + @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_eq(self, target, a, b, pc): + if a == b: + return pc + else: + return target + + @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_ne(self, target, a, b, pc): + if a != b: + return pc + else: + return target + + @arguments("L", "i", "i", "pc", returns="L") def opimpl_goto_if_not_int_gt(self, target, a, b, pc): if a > b: return pc else: return target + @arguments("L", "i", "i", "pc", returns="L") + def opimpl_goto_if_not_int_ge(self, target, a, b, pc): + if a >= b: + return pc + else: + return target + @arguments("L", returns="L") def opimpl_goto(self, target): return target @@ -202,4 +237,21 @@ @arguments("i", "I", "R", returns="i") def opimpl_residual_call_ir_i(self, function, args_i, args_r): # XXX! + assert not args_r + return function(*args_i) + + @arguments("i", "I", "R", returns="r") + def opimpl_residual_call_ir_r(self, function, args_i, args_r): + # XXX! + assert not args_r return function(*args_i) + + @arguments("i", "R", returns="i") + def opimpl_residual_call_r_i(self, function, args_r): + # XXX! + return function(*args_r) + + @arguments("i", "R", returns="r") + def opimpl_residual_call_r_r(self, function, args_r): + # XXX! + return function(*args_r) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu Apr 22 17:22:23 2010 @@ -75,6 +75,7 @@ assert get_stats().aborted_count >= count def meta_interp(self, *args, **kwds): + py.test.skip("XXX") kwds['CPUClass'] = self.CPUClass kwds['type_system'] = self.type_system if "backendopt" not in kwds: From fijal at codespeak.net Thu Apr 22 18:18:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 22 Apr 2010 18:18:24 +0200 (CEST) Subject: [pypy-svn] r73981 - in pypy/trunk/pypy: interpreter interpreter/test module/_stackless module/termios objspace/std Message-ID: <20100422161824.244B2282BAD@codespeak.net> Author: fijal Date: Thu Apr 22 18:18:21 2010 New Revision: 73981 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/interpreter/test/test_objspace.py pypy/trunk/pypy/module/_stackless/interp_coroutine.py pypy/trunk/pypy/module/termios/interp_termios.py pypy/trunk/pypy/objspace/std/objspace.py Log: Kill UnpackValueError. Instead raise OperationError directly. Each caller so far either reraised it as OperationError or forgot and crashed the interpreter Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Thu Apr 22 18:18:21 2010 @@ -209,12 +209,6 @@ def ready(self, result): pass -class UnpackValueError(ValueError): - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - class DescrMismatch(Exception): pass @@ -723,7 +717,8 @@ raise break # done if expected_length != -1 and len(items) == expected_length: - raise UnpackValueError("too many values to unpack") + raise OperationError(space.w_ValueError, + space.wrap("too many values to unpack")) items.append(w_item) if expected_length != -1 and len(items) < expected_length: i = len(items) @@ -731,8 +726,9 @@ plural = "" else: plural = "s" - raise UnpackValueError("need more than %d value%s to unpack" % - (i, plural)) + raise OperationError(space.w_ValueError, + space.wrap("need more than %d value%s to unpack" % + (i, plural))) return items def fixedview(self, w_iterable, expected_length=-1): Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Thu Apr 22 18:18:21 2010 @@ -6,7 +6,7 @@ import sys from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.baseobjspace import UnpackValueError, Wrappable +from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter import gateway, function, eval, pyframe, pytraceback from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name @@ -638,11 +638,7 @@ def UNPACK_SEQUENCE(self, itemcount, next_instr): w_iterable = self.popvalue() - try: - items = self.space.fixedview(w_iterable, itemcount) - except UnpackValueError, e: - w_msg = self.space.wrap(e.msg) - raise OperationError(self.space.w_ValueError, w_msg) + items = self.space.fixedview(w_iterable, itemcount) self.pushrevvalues(itemcount, items) def STORE_ATTR(self, nameindex, next_instr): Modified: pypy/trunk/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_objspace.py (original) +++ pypy/trunk/pypy/interpreter/test/test_objspace.py Thu Apr 22 18:18:21 2010 @@ -61,31 +61,40 @@ assert self.space.newbool(1) == self.space.w_True def test_unpackiterable(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newlist(l) - assert self.space.unpackiterable(w_l) == l - assert self.space.unpackiterable(w_l, 4) == l - raises(ValueError, self.space.unpackiterable, w_l, 3) - raises(ValueError, self.space.unpackiterable, w_l, 5) + w_l = space.newlist(l) + assert space.unpackiterable(w_l) == l + assert space.unpackiterable(w_l, 4) == l + err = raises(OperationError, space.unpackiterable, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.unpackiterable, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_fixedview(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newtuple(l) - assert self.space.fixedview(w_l) == l - assert self.space.fixedview(w_l, 4) == l - raises(ValueError, self.space.fixedview, w_l, 3) - raises(ValueError, self.space.fixedview, w_l, 5) + w_l = space.newtuple(l) + assert space.fixedview(w_l) == l + assert space.fixedview(w_l, 4) == l + err = raises(OperationError, space.fixedview, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.fixedview, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_listview(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newtuple(l) - assert self.space.listview(w_l) == l - assert self.space.listview(w_l, 4) == l - raises(ValueError, self.space.listview, w_l, 3) - raises(ValueError, self.space.listview, w_l, 5) + w_l = space.newtuple(l) + assert space.listview(w_l) == l + assert space.listview(w_l, 4) == l + err = raises(OperationError, space.listview, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.listview, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Thu Apr 22 18:18:21 2010 @@ -15,7 +15,7 @@ experience to decide where to set the limits. """ -from pypy.interpreter.baseobjspace import Wrappable, UnpackValueError +from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w @@ -173,11 +173,8 @@ return nt([w_new_inst, nt(tup_base), nt(tup_state)]) def descr__setstate__(self, space, w_args): - try: - w_flags, w_state, w_thunk, w_parent = space.unpackiterable(w_args, - expected_length=4) - except UnpackValueError, e: - raise OperationError(space.w_ValueError, space.wrap(e.msg)) + w_flags, w_state, w_thunk, w_parent = space.unpackiterable(w_args, + expected_length=4) self.flags = space.int_w(w_flags) if space.is_w(w_parent, space.w_None): w_parent = self.w_getmain(space) @@ -188,11 +185,8 @@ if space.is_w(w_thunk, space.w_None): self.thunk = None else: - try: - w_func, w_args, w_kwds = space.unpackiterable(w_thunk, - expected_length=3) - except UnpackValueError, e: - raise OperationError(space.w_ValueError, space.wrap(e.msg)) + w_func, w_args, w_kwds = space.unpackiterable(w_thunk, + expected_length=3) args = Arguments.frompacked(space, w_args, w_kwds) self.bind(_AppThunk(space, self.costate, w_func, args)) Modified: pypy/trunk/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/trunk/pypy/module/termios/interp_termios.py (original) +++ pypy/trunk/pypy/module/termios/interp_termios.py Thu Apr 22 18:18:21 2010 @@ -28,14 +28,8 @@ return OperationError(w_exception_class, w_exception) def tcsetattr(space, fd, when, w_attributes): - from pypy.interpreter.baseobjspace import UnpackValueError - try: - w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ - space.unpackiterable(w_attributes, expected_length=7) - except UnpackValueError, e: - raise OperationError( - space.w_TypeError, - space.wrap("tcsetattr, arg 3: must be 7 element list")) + w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ + space.unpackiterable(w_attributes, expected_length=7) w_builtin = space.getbuiltinmodule('__builtin__') cc = [] for w_c in space.unpackiterable(w_cc): Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Thu Apr 22 18:18:21 2010 @@ -1,7 +1,7 @@ import __builtin__ import types from pypy.interpreter import pyframe, function, special -from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, UnpackValueError +from pypy.interpreter.baseobjspace import ObjSpace, Wrappable from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.typedef import get_unique_interplevel_subclass from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model, @@ -332,6 +332,11 @@ # have different return type. First one is a resizable list, second # one is not + def _wrap_expected_length(self, expected, got): + return OperationError(self.w_ValueError, + self.wrap("Expected length %d, got %d" % (expected, got))) + + def unpackiterable(self, w_obj, expected_length=-1): if isinstance(w_obj, W_TupleObject): t = w_obj.wrappeditems[:] @@ -340,7 +345,7 @@ else: return ObjSpace.unpackiterable(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def fixedview(self, w_obj, expected_length=-1): @@ -353,7 +358,7 @@ else: return ObjSpace.fixedview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def listview(self, w_obj, expected_length=-1): @@ -364,7 +369,7 @@ else: return ObjSpace.listview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def sliceindices(self, w_slice, w_length): From afa at codespeak.net Thu Apr 22 19:48:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 19:48:04 +0200 (CEST) Subject: [pypy-svn] r73982 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422174804.E16B8282BAD@codespeak.net> Author: afa Date: Thu Apr 22 19:48:03 2010 New Revision: 73982 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Log: Add PyNumber_Check and PyNumber_Long Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Thu Apr 22 19:48:03 2010 @@ -15,6 +15,16 @@ except OperationError: return 0 + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyNumber_Check(space, w_obj): + """Returns 1 if the object o provides numeric protocols, and false otherwise. + This function always succeeds.""" + try: + space.float_w(w_obj) + return 1 + except OperationError: + return 0 + @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PyNumber_AsSsize_t(space, w_obj, w_exc): """Returns o converted to a Py_ssize_t value if o can be interpreted as an @@ -25,7 +35,13 @@ exception is cleared and the value is clipped to PY_SSIZE_T_MIN for a negative integer or PY_SSIZE_T_MAX for a positive integer. """ - return space.int_w(w_obj) #XXX: this is wrong + return space.int_w(w_obj) #XXX: this is wrong on win64 + + at cpython_api([PyObject], PyObject) +def PyNumber_Long(space, w_obj): + """ Returns the o converted to a long integer object on success, or NULL on + failure. This is the equivalent of the Python expression long(o).""" + return space.long(w_obj) def func_rename(newname): return lambda func: func_with_new_name(func, newname) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Thu Apr 22 19:48:03 2010 @@ -4,10 +4,22 @@ from pypy.module.cpyext import sequence class TestIterator(BaseApiTest): - def test_index(self, space, api): + def test_check(self, space, api): assert api.PyIndex_Check(space.wrap(12)) + assert api.PyIndex_Check(space.wrap(-12L)) + assert not api.PyIndex_Check(space.wrap(12.1)) assert not api.PyIndex_Check(space.wrap('12')) + assert api.PyNumber_Check(space.wrap(12)) + assert api.PyNumber_Check(space.wrap(-12L)) + assert api.PyNumber_Check(space.wrap(12.1)) + assert not api.PyNumber_Check(space.wrap('12')) + assert not api.PyNumber_Check(space.wrap(1+3j)) + + def test_number_long(self, space, api): + w_l = api.PyNumber_Long(space.wrap(123)) + assert api.PyLong_CheckExact(w_l) + def test_numbermethods(self, space, api): assert "ab" == space.unwrap( api.PyNumber_Add(space.wrap("a"), space.wrap("b"))) From afa at codespeak.net Thu Apr 22 19:55:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 19:55:18 +0200 (CEST) Subject: [pypy-svn] r73983 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100422175518.D6F2B282BAD@codespeak.net> Author: afa Date: Thu Apr 22 19:55:17 2010 New Revision: 73983 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Log: Provide stubs for many functions around PyThreadState and PyInterpreterState, and move into pystate.py functions that are empty, but probably already right. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pystate.h Thu Apr 22 19:55:17 2010 @@ -1,14 +1,17 @@ #ifndef Py_PYSTATE_H #define Py_PYSTATE_H -typedef struct _ts { - int initialized; // not used -} PyThreadState; +struct _ts; /* Forward */ +struct _is; /* Forward */ typedef struct _is { int _foo; } PyInterpreterState; +typedef struct _ts { + PyInterpreterState *interp; +} PyThreadState; + #define Py_BEGIN_ALLOW_THREADS { \ PyThreadState *_save; \ _save = PyEval_SaveThread(); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Thu Apr 22 19:55:17 2010 @@ -1,9 +1,10 @@ from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ - cpython_struct, PyGILState_STATE + cpython_struct from pypy.rpython.lltypesystem import rffi, lltype PyThreadState = lltype.Ptr(cpython_struct("PyThreadState", ())) +PyInterpreterState = lltype.Ptr(cpython_struct("PyInterpreterState", ())) @cpython_api([], PyThreadState, error=CANNOT_FAIL) def PyEval_SaveThread(space): @@ -28,11 +29,12 @@ from pypy.module.thread.gil import after_external_call after_external_call() - at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) -def PyGILState_Ensure(space): - return 0 - - at cpython_api([PyGILState_STATE], lltype.Void) -def PyGILState_Release(space, state): + at cpython_api([], lltype.Void) +def PyEval_InitThreads(space): return + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) +def PyEval_ThreadsInitialized(space): + return 1 + + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubsactive.py Thu Apr 22 19:55:17 2010 @@ -1,6 +1,8 @@ from pypy.module.cpyext.pyobject import PyObject -from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL +from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL, CConfig from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.pystate import PyThreadState, PyInterpreterState + @cpython_api([PyObject], rffi.VOIDP, error=CANNOT_FAIL) #XXX def PyFile_AsFile(space, p): @@ -44,3 +46,68 @@ instead of the repr().""" raise NotImplementedError + at cpython_api([], lltype.Void) +def PyErr_Print(space): + """Alias for PyErr_PrintEx(1).""" + raise NotImplementedError + + at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) +def PyThreadState_New(space, interp): + """Create a new thread state object belonging to the given interpreter object. + The global interpreter lock need not be held, but may be held if it is + necessary to serialize calls to this function.""" + raise NotImplementedError + + at cpython_api([PyThreadState], lltype.Void) +def PyThreadState_Clear(space, tstate): + """Reset all information in a thread state object. The global interpreter lock + must be held.""" + raise NotImplementedError + + at cpython_api([PyThreadState], lltype.Void) +def PyThreadState_Delete(space, tstate): + """Destroy a thread state object. The global interpreter lock need not be held. + The thread state must have been reset with a previous call to + PyThreadState_Clear().""" + raise NotImplementedError + + at cpython_api([PyThreadState], PyThreadState, error=CANNOT_FAIL) +def PyThreadState_Swap(space, tstate): + """Swap the current thread state with the thread state given by the argument + tstate, which may be NULL. The global interpreter lock must be held.""" + raise NotImplementedError + + at cpython_api([PyThreadState], lltype.Void) +def PyEval_AcquireThread(space, tstate): + """Acquire the global interpreter lock and set the current thread state to + tstate, which should not be NULL. The lock must have been created earlier. + If this thread already has the lock, deadlock ensues. This function is not + available when thread support is disabled at compile time.""" + raise NotImplementedError + + at cpython_api([PyThreadState], lltype.Void) +def PyEval_ReleaseThread(space, tstate): + """Reset the current thread state to NULL and release the global interpreter + lock. The lock must have been created earlier and must be held by the current + thread. The tstate argument, which must not be NULL, is only used to check + that it represents the current thread state --- if it isn't, a fatal error is + reported. This function is not available when thread support is disabled at + compile time.""" + raise NotImplementedError + + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) +def Py_MakePendingCalls(space): + return 0 + +PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', + typedef='PyGILState_STATE', + compilation_info=CConfig._compilation_info_) + + at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) +def PyGILState_Ensure(space): + return 0 + + at cpython_api([PyGILState_STATE], lltype.Void) +def PyGILState_Release(space, state): + return + From afa at codespeak.net Thu Apr 22 19:55:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 19:55:44 +0200 (CEST) Subject: [pypy-svn] r73984 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422175544.B6BF0282BAD@codespeak.net> Author: afa Date: Thu Apr 22 19:55:43 2010 New Revision: 73984 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Belongs to the previous commit. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 22 19:55:43 2010 @@ -63,10 +63,6 @@ assert CONST_STRING is not rffi.CCHARP assert CONST_WSTRING is not rffi.CWCHARP -PyGILState_STATE = rffi.COpaquePtr('PyGILState_STATE', - typedef='PyGILState_STATE', - compilation_info=CConfig._compilation_info_) - constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING METH_COEXIST METH_STATIC METH_CLASS From afa at codespeak.net Thu Apr 22 20:02:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:02:58 +0200 (CEST) Subject: [pypy-svn] r73985 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src Message-ID: <20100422180258.A27FA282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:02:52 2010 New Revision: 73985 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c Log: Add and/or expose the implementations of PyObject_AsWriteBuffer and PyBuffer_New, copied from CPython. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 22 20:02:52 2010 @@ -251,8 +251,9 @@ 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', - 'PyBuffer_FromMemory', 'PyBuffer_Type', 'init_bufferobject', - '_PyArg_NoKeywords', 'PyObject_AsReadBuffer', 'PyObject_CheckReadBuffer', + 'PyBuffer_FromMemory', 'PyBuffer_New', 'PyBuffer_Type', 'init_bufferobject', + '_PyArg_NoKeywords', + 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Thu Apr 22 20:02:52 2010 @@ -467,6 +467,7 @@ /* Copied from CPython ----------------------------- */ int PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *); +int PyObject_AsWriteBuffer(PyObject *, void **, Py_ssize_t *); int PyObject_CheckReadBuffer(PyObject *); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/object.c Thu Apr 22 20:02:52 2010 @@ -44,6 +44,39 @@ return 0; } +int PyObject_AsWriteBuffer(PyObject *obj, + void **buffer, + Py_ssize_t *buffer_len) +{ + PyBufferProcs *pb; + void*pp; + Py_ssize_t len; + + if (obj == NULL || buffer == NULL || buffer_len == NULL) { + null_error(); + return -1; + } + pb = obj->ob_type->tp_as_buffer; + if (pb == NULL || + pb->bf_getwritebuffer == NULL || + pb->bf_getsegcount == NULL) { + PyErr_SetString(PyExc_TypeError, + "expected a writeable buffer object"); + return -1; + } + if ((*pb->bf_getsegcount)(obj, NULL) != 1) { + PyErr_SetString(PyExc_TypeError, + "expected a single-segment buffer object"); + return -1; + } + len = (*pb->bf_getwritebuffer)(obj,0,&pp); + if (len < 0) + return -1; + *buffer = pp; + *buffer_len = len; + return 0; +} + int PyObject_CheckReadBuffer(PyObject *obj) { From afa at codespeak.net Thu Apr 22 20:12:14 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:12:14 +0200 (CEST) Subject: [pypy-svn] r73986 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100422181214.2C18E282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:12:11 2010 New Revision: 73986 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: add some 'extern "C"' statements for the benefit of extensions written in C++ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 22 20:12:11 2010 @@ -589,6 +589,9 @@ functions = [] pypy_decls = [] pypy_decls.append("#ifndef PYPY_STANDALONE\n") + pypy_decls.append("#ifdef __cplusplus") + pypy_decls.append("extern \"C\" {") + pypy_decls.append("#endif\n") for decl in FORWARD_DECLS: pypy_decls.append("%s;" % (decl,)) @@ -628,6 +631,10 @@ pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name_clean)) if not globals_are_pointers and "#" not in name: pypy_decls.append("#define %s (PyObject*)&%s" % (name, name,)) + + pypy_decls.append("#ifdef __cplusplus") + pypy_decls.append("}") + pypy_decls.append("#endif") pypy_decls.append("#endif /*PYPY_STANDALONE*/\n") pypy_decl_h = udir.join('pypy_decl.h') Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Thu Apr 22 20:12:11 2010 @@ -3,6 +3,10 @@ #include +#ifdef __cplusplus +extern "C" { +#endif + #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None @@ -476,4 +480,7 @@ #define PyObject_Length PyObject_Size +#ifdef __cplusplus +} #endif +#endif /* !Py_OBJECT_H */ From afa at codespeak.net Thu Apr 22 20:19:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:19:00 +0200 (CEST) Subject: [pypy-svn] r73987 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422181900.59F0C282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:18:58 2010 New Revision: 73987 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: PyLong_AsLong, PyLong_FromDouble Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Thu Apr 22 20:18:58 2010 @@ -32,6 +32,14 @@ raised.""" return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + at cpython_api([PyObject], lltype.Signed, error=-1) +def PyLong_AsLong(space, w_long): + """ + Return a C long representation of the contents of pylong. If + pylong is greater than LONG_MAX, an OverflowError is raised + and -1 will be returned.""" + return space.int_w(w_long) + @cpython_api([PyObject], rffi.LONGLONG, error=-1) def PyLong_AsLongLong(space, w_long): """ @@ -48,6 +56,11 @@ raised.""" return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + at cpython_api([lltype.Float], PyObject) +def PyLong_FromDouble(space, val): + """Return a new PyLongObject object from v, or NULL on failure.""" + return space.long(space.wrap(val)) + @cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject) def PyLong_FromString(space, str, pend, base): """Return a new PyLongObject based on the string value in str, which is Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Thu Apr 22 20:18:58 2010 @@ -16,12 +16,25 @@ value = api.PyLong_FromLong(sys.maxint + 1) assert isinstance(value, W_LongObject) assert space.unwrap(value) == sys.maxint + 1 # should obviously fail but doesnt - - def test_asulong(self, space, api): + + def test_aslong(self, space, api): w_value = api.PyLong_FromLong((sys.maxint - 1) / 2) - w_value = space.mul(w_value, space.wrap(4)) + + w_value = space.mul(w_value, space.wrap(2)) + value = api.PyLong_AsLong(w_value) + assert value == (sys.maxint - 1) + + w_value = space.mul(w_value, space.wrap(2)) + + value = api.PyLong_AsLong(w_value) + assert value == -1 and api.PyErr_Occurred() is space.w_OverflowError + api.PyErr_Clear() value = api.PyLong_AsUnsignedLong(w_value) assert value == (sys.maxint - 1) * 2 + + def test_fromdouble(self, space, api): + w_value = api.PyLong_FromDouble(-12.74) + assert space.unwrap(w_value) == -12 def test_type_check(self, space, api): w_l = space.wrap(sys.maxint + 1) From afa at codespeak.net Thu Apr 22 20:23:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:23:04 +0200 (CEST) Subject: [pypy-svn] r73988 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422182304.454FC282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:23:02 2010 New Revision: 73988 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_Str Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Thu Apr 22 20:23:02 2010 @@ -146,6 +146,10 @@ return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise @cpython_api([PyObject], PyObject) +def PyObject_Str(space, w_obj): + return space.str(w_obj) + + at cpython_api([PyObject], PyObject) def PyObject_Repr(space, w_obj): """Compute a string representation of object o. Returns the string representation on success, NULL on failure. This is the equivalent of the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Thu Apr 22 20:23:02 2010 @@ -73,6 +73,11 @@ def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" + assert space.str_w(api.PyObject_Repr(space.wrap("a"))) == "'a'" + + w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" + assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" def test_RichCompare(self, space, api): def compare(w_o1, w_o2, opid): From afa at codespeak.net Thu Apr 22 20:28:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:28:21 +0200 (CEST) Subject: [pypy-svn] r73989 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422182821.3DAE4282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:28:20 2010 New Revision: 73989 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Log: PyDict_DelItem Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Thu Apr 22 20:28:20 2010 @@ -26,6 +26,14 @@ else: PyErr_BadInternalCall(space) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyDict_DelItem(space, w_dict, w_key): + if PyDict_Check(space, w_dict): + space.delitem(w_dict, w_key) + return 0 + else: + PyErr_BadInternalCall(space) + @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyDict_SetItemString(space, w_dict, key_ptr, w_obj): if PyDict_Check(space, w_dict): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Thu Apr 22 20:28:20 2010 @@ -22,6 +22,12 @@ assert api.PyErr_Occurred() is space.w_KeyError api.PyErr_Clear() + assert api.PyDict_DelItem(d, space.wrap("c")) == 0 + assert api.PyDict_DelItem(d, space.wrap("name")) < 0 + assert api.PyErr_Occurred() is space.w_KeyError + api.PyErr_Clear() + assert api.PyDict_Size(d) == 0 + def test_check(self, space, api): d = api.PyDict_New() assert api.PyDict_Check(d) From afa at codespeak.net Thu Apr 22 20:33:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:33:15 +0200 (CEST) Subject: [pypy-svn] r73990 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422183315.385F4282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:33:13 2010 New Revision: 73990 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Log: fix inconsitency between setitem and delitem: always check the exception when -1 is returned Modified: pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/slotdefs.py Thu Apr 22 20:33:13 2010 @@ -69,7 +69,7 @@ index = space.int_w(space.index(args_w[0])) res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception() + space.fromcache(State).check_and_raise_exception(always=True) def wrap_sq_delitem(space, w_self, w_args, func): func_target = rffi.cast(ssizeobjargproc, func) From afa at codespeak.net Thu Apr 22 20:38:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:38:30 +0200 (CEST) Subject: [pypy-svn] r73991 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100422183830.DEADC282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:38:29 2010 New Revision: 73991 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Log: Don't use hardcoded sizes that will be wrong on 64bit Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_stringobject.py Thu Apr 22 20:38:29 2010 @@ -28,7 +28,7 @@ if(PyString_Size(s) == 11) { result = 1; } - if(s->ob_type->tp_basicsize != 16) + if(s->ob_type->tp_basicsize != sizeof(void*)*4) result = 0; Py_DECREF(s); return PyBool_FromLong(result); From afa at codespeak.net Thu Apr 22 20:42:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 20:42:30 +0200 (CEST) Subject: [pypy-svn] r73992 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422184230.0B2BD282BAD@codespeak.net> Author: afa Date: Thu Apr 22 20:42:28 2010 New Revision: 73992 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Log: PyErr_Fetch and PyErr_Restore Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Thu Apr 22 20:42:28 2010 @@ -4,7 +4,8 @@ from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning -from pypy.module.cpyext.pyobject import PyObject, make_ref, register_container +from pypy.module.cpyext.pyobject import ( + PyObject, PyObjectP, make_ref, Py_DecRef, register_container) from pypy.module.cpyext.state import State from pypy.rlib.rposix import get_errno @@ -60,6 +61,43 @@ state = space.fromcache(State) state.clear_exception() + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_Fetch(space, ptype, pvalue, ptraceback): + """Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to NULL. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be NULL even when the type object is not. + + This function is normally only used by code that needs to handle exceptions or + by code that needs to save and restore the error indicator temporarily.""" + state = space.fromcache(State) + ptype[0] = make_ref(space, state.exc_type, steal=True) + pvalue[0] = make_ref(space, state.exc_type, steal=True) + state.exc_type = None + state.exc_value = None + ptraceback[0] = lltype.nullptr(PyObject.TO) + + at cpython_api([PyObject, PyObject, PyObject], lltype.Void) +def PyErr_Restore(space, w_type, w_value, w_traceback): + """Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are NULL, the error + indicator is cleared. Do not pass a NULL type and non-NULL value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + This function is normally only used by code that needs to save and restore the + error indicator temporarily; use PyErr_Fetch() to save the current + exception state.""" + state = space.fromcache(State) + state.exc_type = w_type + state.exc_value = w_value + Py_DecRef(space, w_type) + Py_DecRef(space, w_value) + @cpython_api([], lltype.Void) def PyErr_BadInternalCall(space): raise OperationError(space.w_SystemError, space.wrap("Bad internal call!")) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Thu Apr 22 20:42:28 2010 @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ - PyObject, ADDR,\ + PyObject, PyObjectP, ADDR,\ Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pyerrors.py Thu Apr 22 20:42:28 2010 @@ -94,6 +94,29 @@ ]) module.check_error() + def test_fetch_and_restore(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (PyErr_Occurred()) + return NULL; + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != type) + Py_RETURN_FALSE; + PyErr_Restore(type, val, tb); + if (!PyErr_Occurred()) + Py_RETURN_FALSE; + PyErr_Clear(); + Py_RETURN_TRUE; + ''' + ), + ]) + module.check_error() def test_SetFromErrno(self): skip("The test does not set the errno in a way which " From afa at codespeak.net Thu Apr 22 21:34:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 21:34:59 +0200 (CEST) Subject: [pypy-svn] r73993 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100422193459.54B8D282BAD@codespeak.net> Author: afa Date: Thu Apr 22 21:34:56 2010 New Revision: 73993 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c Log: Fix test: Py_SIZE is supposed to exist since python2.6 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c Thu Apr 22 21:34:56 2010 @@ -15,6 +15,9 @@ #endif /* HAVE_SYS_TYPES_H */ #endif /* !STDC_HEADERS */ +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + struct arrayobject; /* Forward */ /* All possible arraydescr values are defined in the vector "descriptors" From afa at codespeak.net Thu Apr 22 21:45:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 21:45:21 +0200 (CEST) Subject: [pypy-svn] r73994 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100422194521.79C93282BAD@codespeak.net> Author: afa Date: Thu Apr 22 21:45:18 2010 New Revision: 73994 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c pypy/branch/cpython-extension/pypy/module/cpyext/test/test_borrow.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py Log: Fix the tests I broke by removing Py_TYPE &co Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/_sre.c Thu Apr 22 21:45:18 2010 @@ -81,6 +81,9 @@ #define PyObject_DEL(op) PyMem_DEL((op)) #endif +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + /* -------------------------------------------------------------------- */ #if defined(_MSC_VER) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/foo.c Thu Apr 22 21:45:18 2010 @@ -264,7 +264,7 @@ { PyObject *m, *d; - Py_TYPE(&footype) = &PyType_Type; + footype.ob_type = &PyType_Type; /* Workaround for quirk in Visual Studio, see */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_borrow.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_borrow.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_borrow.py Thu Apr 22 21:45:18 2010 @@ -19,7 +19,7 @@ state.print_refcounts() py.test.raises(AssertionError, api.Py_DecRef, one_pyo) -class AppTestStringObject(AppTestCpythonExtensionBase): +class AppTestBorrow(AppTestCpythonExtensionBase): def test_tuple_borrowing(self): module = self.import_extension('foo', [ ("test_borrowing", "METH_NOARGS", @@ -27,13 +27,13 @@ PyObject *t = PyTuple_New(1); PyObject *f = PyFloat_FromDouble(42.0); PyObject *g = NULL; - printf("Refcnt1: %i\\n", Py_REFCNT(f)); + printf("Refcnt1: %i\\n", f->ob_refcnt); PyTuple_SetItem(t, 0, f); // steals reference - printf("Refcnt2: %i\\n", Py_REFCNT(f)); + printf("Refcnt2: %i\\n", f->ob_refcnt); f = PyTuple_GetItem(t, 0); // borrows reference - printf("Refcnt3: %i\\n", Py_REFCNT(f)); + printf("Refcnt3: %i\\n", f->ob_refcnt); g = PyTuple_GetItem(t, 0); // borrows reference again - printf("Refcnt4: %i\\n", Py_REFCNT(f)); + printf("Refcnt4: %i\\n", f->ob_refcnt); printf("COMPARE: %i\\n", f == g); Py_DECREF(t); Py_RETURN_TRUE; Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Thu Apr 22 21:45:18 2010 @@ -421,12 +421,12 @@ static PyObject* foo_pi(PyObject* self, PyObject *args) { PyObject *true = Py_True; - int refcnt = Py_REFCNT(true); + int refcnt = true->ob_refcnt; int refcnt_after; Py_INCREF(true); Py_INCREF(true); PyBool_Check(true); - refcnt_after = Py_REFCNT(true); + refcnt_after = true->ob_refcnt; Py_DECREF(true); Py_DECREF(true); fprintf(stderr, "REFCNT %i %i\\n", refcnt, refcnt_after); @@ -436,14 +436,14 @@ { PyObject *true = Py_True; PyObject *tup = NULL; - int refcnt = Py_REFCNT(true); + int refcnt = true->ob_refcnt; int refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_after = Py_REFCNT(true); + refcnt_after = true->ob_refcnt; Py_DECREF(tup); fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); return PyBool_FromLong(refcnt_after == refcnt); Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pystate.py Thu Apr 22 21:45:18 2010 @@ -1,6 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -class AppTestStringObject(AppTestCpythonExtensionBase): +class AppTestThreads(AppTestCpythonExtensionBase): def test_allow_threads(self): module = self.import_extension('foo', [ ("test", "METH_NOARGS", From benjamin at codespeak.net Thu Apr 22 22:26:02 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 22 Apr 2010 22:26:02 +0200 (CEST) Subject: [pypy-svn] r73995 - pypy/trunk/pypy/objspace/std Message-ID: <20100422202602.BFCF6282BAD@codespeak.net> Author: benjamin Date: Thu Apr 22 22:26:01 2010 New Revision: 73995 Modified: pypy/trunk/pypy/objspace/std/objspace.py Log: kill empty line Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Thu Apr 22 22:26:01 2010 @@ -335,7 +335,6 @@ def _wrap_expected_length(self, expected, got): return OperationError(self.w_ValueError, self.wrap("Expected length %d, got %d" % (expected, got))) - def unpackiterable(self, w_obj, expected_length=-1): if isinstance(w_obj, W_TupleObject): From afa at codespeak.net Thu Apr 22 23:30:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 23:30:30 +0200 (CEST) Subject: [pypy-svn] r73997 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100422213030.5389E282BAD@codespeak.net> Author: afa Date: Thu Apr 22 23:30:28 2010 New Revision: 73997 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Log: Add PyRun_String Modified: pypy/branch/cpython-extension/pypy/module/cpyext/eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/eval.py Thu Apr 22 23:30:28 2010 @@ -1,5 +1,7 @@ - -from pypy.module.cpyext.api import cpython_api, PyObject, CANNOT_FAIL +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.api import ( + cpython_api, PyObject, CANNOT_FAIL, CONST_STRING) @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): @@ -25,3 +27,29 @@ success, or NULL on failure. This is the equivalent of the Python expression apply(callable_object, args, kw) or callable_object(*args, **kw).""" return space.call(w_obj, w_args, w_kw) + +# These constants are also defined in include/eval.h +Py_single_input = 256 +Py_file_input = 257 +Py_eval_input = 258 + + at cpython_api([CONST_STRING, rffi.INT_real, PyObject, PyObject], PyObject) +def PyRun_String(space, str, start, w_globals, w_locals): + """This is a simplified interface to PyRun_StringFlags() below, leaving + flags set to NULL.""" + from pypy.module.__builtin__ import compiling + w_source = space.wrap(rffi.charp2str(str)) + filename = "" + start = rffi.cast(lltype.Signed, start) + if start == Py_file_input: + mode = 'exec' + elif start == Py_eval_input: + mode = 'eval' + elif start == Py_single_input: + mode = 'single' + else: + raise OperationError(space.w_ValueError, space.wrap( + "invalid mode parameter for PyRun_String")) + w_code = compiling.compile(space, w_source, filename, mode) + return compiling.eval(space, w_code, w_globals, w_locals) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h Thu Apr 22 23:30:28 2010 @@ -17,6 +17,11 @@ PyObject * PyObject_CallFunction(PyObject *obj, char *format, ...); PyObject * PyObject_CallMethod(PyObject *obj, char *name, char *format, ...); +/* These constants are also defined in cpyext/eval.py */ +#define Py_single_input 256 +#define Py_file_input 257 +#define Py_eval_input 258 + #ifdef __cplusplus } #endif Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_eval.py Thu Apr 22 23:30:28 2010 @@ -1,5 +1,7 @@ +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.eval import Py_single_input, Py_file_input, Py_eval_input class TestEval(BaseApiTest): def test_eval(self, space, api): @@ -56,6 +58,25 @@ assert space.int_w(w_res) == 10 + def test_run_string(self, space, api): + def run(code, start, w_globals, w_locals): + buf = rffi.str2charp(code) + try: + return api.PyRun_String(buf, start, w_globals, w_locals) + finally: + rffi.free_charp(buf) + + w_globals = space.newdict() + assert 42 * 43 == space.unwrap( + run("42 * 43", Py_eval_input, w_globals, w_globals)) + assert api.PyObject_Size(w_globals) == 0 + + assert run("a = 42 * 43", Py_single_input, + w_globals, w_globals) == space.w_None + assert 42 * 43 == space.unwrap( + api.PyObject_GetItem(w_globals, space.wrap("a"))) + + class AppTestCall(AppTestCpythonExtensionBase): def test_CallFunction(self): module = self.import_extension('foo', [ From afa at codespeak.net Thu Apr 22 23:54:52 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Apr 2010 23:54:52 +0200 (CEST) Subject: [pypy-svn] r73998 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100422215452.25420282BAD@codespeak.net> Author: afa Date: Thu Apr 22 23:54:50 2010 New Revision: 73998 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: Add PyObject_GenericSetAttr I don't know how to test this function: faked objects don't support setattr. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Thu Apr 22 23:54:50 2010 @@ -214,6 +214,18 @@ w_descr = object_getattribute(space) return space.get_and_call_function(w_descr, w_obj, w_name) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_GenericSetAttr(space, w_obj, w_name, w_value): + """Generic attribute setter function that is meant to be put into a type + object's tp_setattro slot. It looks for a data descriptor in the + dictionary of classes in the object's MRO, and if found it takes preference + over setting the attribute in the instance dictionary. Otherwise, the + attribute is set in the object's __dict__ (if present). Otherwise, + an AttributeError is raised and -1 is returned.""" + from pypy.objspace.descroperation import object_setattr + w_descr = object_setattr(space) + return space.get_and_call_function(w_descr, w_obj, w_name, w_value) + @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_IsInstance(space, w_inst, w_cls): """Returns 1 if inst is an instance of the class cls or a subclass of From afa at codespeak.net Fri Apr 23 00:29:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 00:29:39 +0200 (CEST) Subject: [pypy-svn] r73999 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422222939.57280282BAD@codespeak.net> Author: afa Date: Fri Apr 23 00:29:37 2010 New Revision: 73999 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_GetSize, PyUnicode_EncodeMBCS and PyUnicode_DecodeMBCS on windows the detection of leaks is not perfect, see test_unicodeobject.test_leak. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Fri Apr 23 00:29:37 2010 @@ -2,10 +2,12 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.unicodeobject import Py_UNICODE from pypy.rpython.lltypesystem import rffi, lltype +import sys, py class TestUnicode(BaseApiTest): def test_unicodeobject(self, space, api): assert api.PyUnicode_GET_SIZE(space.wrap(u'sp?m')) == 4 + assert api.PyUnicode_GetSize(space.wrap(u'sp?m')) == 4 unichar = rffi.sizeof(Py_UNICODE) assert api.PyUnicode_GET_DATA_SIZE(space.wrap(u'sp?m')) == 4 * unichar @@ -94,3 +96,27 @@ api.PyUnicode_Decode(b_text, 4, b_encoding, None)) == u'caf\xe9' rffi.free_charp(b_text) rffi.free_charp(b_encoding) + + def test_leak(self): + py.test.skip("This test seems to leak memory") + size = 50 + raw_buf, gc_buf = rffi.alloc_buffer(size) + for i in range(size): raw_buf[i] = 'a' + str = rffi.str_from_buffer(raw_buf, gc_buf, size, size) + rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + + def test_mbcs(self, space, api): + if sys.platform != 'win32': + py.test.skip("mcbs encoding only exists on Windows") + # unfortunately, mbcs is locale-dependent. + # This tests works at least on a Western Windows. + unichars = u"abc" + unichr(12345) + wbuf = rffi.unicode2wcharp(unichars) + w_str = api.PyUnicode_EncodeMBCS(wbuf, 4, None) + rffi.free_wcharp(wbuf) + assert space.type(w_str) is space.w_str + assert space.str_w(w_str) == "abc?" + + # XXX this test seems to leak references, see test_leak above + from pypy.module.cpyext.test.test_cpyext import freeze_refcnts + freeze_refcnts(self) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Fri Apr 23 00:29:37 2010 @@ -10,6 +10,7 @@ from pypy.module.cpyext.stringobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.objspace.std import unicodeobject, unicodetype +import sys PyUnicodeObjectStruct = lltype.ForwardReference() PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) @@ -142,6 +143,15 @@ space.wrap("expected unicode object")) return PyUnicode_AS_UNICODE(space, ref) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyUnicode_GetSize(space, ref): + if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode: + ref = rffi.cast(PyUnicodeObject, ref) + return ref.c_size + else: + w_obj = from_ref(space, ref) + return space.int_w(space.len(w_obj)) + @cpython_api([PyUnicodeObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1) def PyUnicode_AsWideChar(space, ref, buf, size): """Copy the Unicode object contents into the wchar_t buffer w. At most @@ -245,3 +255,31 @@ else: w_errors = space.w_None return space.call_method(w_str, 'decode', w_encoding, w_errors) + + +if sys.platform == 'win32': + @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject) + def PyUnicode_EncodeMBCS(space, wchar_p, length, errors): + """Encode the Py_UNICODE buffer of the given size using MBCS and return a + Python string object. Return NULL if an exception was raised by the codec. + """ + w_unicode = space.wrap(rffi.wcharpsize2unicode(wchar_p, length)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + else: + w_errors = space.w_None + return space.call_method(w_unicode, "encode", + space.wrap("mbcs"), w_errors) + + @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject) + def PyUnicode_DecodeMBCS(space, s, size, errors): + """Create a Unicode object by decoding size bytes of the MBCS encoded string s. + Return NULL if an exception was raised by the codec. + """ + w_str = space.wrap(rffi.charpsize2str(s, size)) + w_encoding = space.wrap("mbcs") + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + else: + w_errors = space.w_None + return space.call_method(w_str, 'decode', w_encoding, w_errors) From afa at codespeak.net Fri Apr 23 00:32:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 00:32:09 +0200 (CEST) Subject: [pypy-svn] r74000 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100422223209.97152282BAD@codespeak.net> Author: afa Date: Fri Apr 23 00:32:07 2010 New Revision: 74000 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Log: The Py_PRINT_RAW constant, used when calling PyObject_Print Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/object.h Fri Apr 23 00:32:07 2010 @@ -300,8 +300,11 @@ } PyTypeObject; +/* Flag bits for printing: */ +#define Py_PRINT_RAW 1 /* No string quotes etc. */ + /* -`Type flags (tp_flags) +Type flags (tp_flags) These flags are used to extend the type structure in a backwards-compatible fashion. Extensions can use the flags to indicate (and test) when a given From afa at codespeak.net Fri Apr 23 00:40:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 00:40:43 +0200 (CEST) Subject: [pypy-svn] r74001 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100422224043.D4877282BAD@codespeak.net> Author: afa Date: Fri Apr 23 00:40:41 2010 New Revision: 74001 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: _Py_HashPointer Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Fri Apr 23 00:40:41 2010 @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root, SpaceCache from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, bootstrap_function, \ - PyObject, PyObjectP, ADDR,\ + PyObject, PyObjectP, ADDR, CANNOT_FAIL, \ Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject @@ -315,4 +315,8 @@ obj_ptr = rffi.cast(ADDR, obj) borrowees[obj_ptr] = None +#___________________________________________________________ + at cpython_api([rffi.VOIDP_real], lltype.Signed, error=CANNOT_FAIL) +def _Py_HashPointer(space, ptr): + return rffi.cast(lltype.Signed, ptr) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Fri Apr 23 00:40:41 2010 @@ -541,3 +541,15 @@ ), ]) raises(SystemError, mod.newexc, "name", Exception, {}) + + def test_hash_pointer(self): + mod = self.import_extension('foo', [ + ('get_hash', 'METH_NOARGS', + ''' + return PyInt_FromLong(_Py_HashPointer(Py_None)); + ''' + ), + ]) + h = mod.get_hash() + assert h != 0 + assert h % 4 == 0 # it's the pointer value From afa at codespeak.net Fri Apr 23 00:50:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 00:50:25 +0200 (CEST) Subject: [pypy-svn] r74002 - pypy/branch/cpython-extension/pypy/rpython/lltypesystem Message-ID: <20100422225025.3C697282BAD@codespeak.net> Author: afa Date: Fri Apr 23 00:50:23 2010 New Revision: 74002 Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py Log: Don't count the creation of _subarray pointers as memory allocation Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py Fri Apr 23 00:50:23 2010 @@ -1587,7 +1587,7 @@ _cache = weakref.WeakKeyDictionary() # parentarray -> {subarrays} def __init__(self, TYPE, parent, baseoffset_or_fieldname): - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation=False) self._setparentstructure(parent, baseoffset_or_fieldname) # Keep the parent array alive, we share the same allocation. # Don't do it if we are inside a GC object, though -- it's someone From benjamin at codespeak.net Fri Apr 23 02:41:41 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 23 Apr 2010 02:41:41 +0200 (CEST) Subject: [pypy-svn] r74003 - in pypy/branch/cpython-extension: . dotviewer lib-python py/impl/code pypy pypy/annotation pypy/config pypy/interpreter pypy/interpreter/astcompiler pypy/interpreter/astcompiler/test pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit/backend/cli/test pypy/jit/backend/llvm/test pypy/jit/backend/x86/test pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl/tinyframe pypy/module/__builtin__ pypy/module/_codecs pypy/module/_file pypy/module/_file/test pypy/module/_locale pypy/module/_locale/test pypy/module/_stackless pypy/module/bz2 pypy/module/exceptions pypy/module/posix pypy/module/posix/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/signal pypy/module/sys pypy/module/sys/test pypy/module/termios pypy/objspace/flow pypy/objspace/flow/test pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory/gctransform pypy/rpython/test pypy/tool pypy/tool/pytest pypy/translator pypy/translator/c pypy/translator/c/src pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/jvm pypy/translator/tool pypy/translator/tool/test Message-ID: <20100423004141.A5BB4282BAD@codespeak.net> Author: benjamin Date: Fri Apr 23 02:41:35 2010 New Revision: 74003 Added: pypy/branch/cpython-extension/pypy/jit/tl/tinyframe/ - copied from r74002, pypy/trunk/pypy/jit/tl/tinyframe/ pypy/branch/cpython-extension/pypy/rlib/rlocale.py - copied unchanged from r74002, pypy/trunk/pypy/rlib/rlocale.py pypy/branch/cpython-extension/pypy/rlib/test/test_rlocale.py - copied unchanged from r74002, pypy/trunk/pypy/rlib/test/test_rlocale.py pypy/branch/cpython-extension/pypy/translator/goal/test2/test_targetpypy.py - copied unchanged from r74002, pypy/trunk/pypy/translator/goal/test2/test_targetpypy.py Removed: pypy/branch/cpython-extension/pypy/module/_locale/app_locale.py pypy/branch/cpython-extension/pypy/rlib/rsre/_rsre_platform.py pypy/branch/cpython-extension/pypy/rlib/rsre/test/test_rsre_platform.py pypy/branch/cpython-extension/pypy/translator/goal/buildcache2.py Modified: pypy/branch/cpython-extension/ (props changed) pypy/branch/cpython-extension/dotviewer/ (props changed) pypy/branch/cpython-extension/lib-python/ (props changed) pypy/branch/cpython-extension/py/impl/code/_assertionold.py pypy/branch/cpython-extension/pypy/ (props changed) pypy/branch/cpython-extension/pypy/annotation/annrpython.py pypy/branch/cpython-extension/pypy/config/translationoption.py pypy/branch/cpython-extension/pypy/conftest.py pypy/branch/cpython-extension/pypy/interpreter/astcompiler/assemble.py pypy/branch/cpython-extension/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py pypy/branch/cpython-extension/pypy/interpreter/function.py pypy/branch/cpython-extension/pypy/interpreter/gateway.py pypy/branch/cpython-extension/pypy/interpreter/pycode.py pypy/branch/cpython-extension/pypy/interpreter/pycompiler.py pypy/branch/cpython-extension/pypy/interpreter/pyframe.py pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py pypy/branch/cpython-extension/pypy/interpreter/pyparser/metaparser.py pypy/branch/cpython-extension/pypy/interpreter/pyparser/parser.py pypy/branch/cpython-extension/pypy/interpreter/pyparser/pyparse.py pypy/branch/cpython-extension/pypy/interpreter/pyparser/test/test_metaparser.py pypy/branch/cpython-extension/pypy/interpreter/test/test_appinterp.py pypy/branch/cpython-extension/pypy/interpreter/test/test_compiler.py pypy/branch/cpython-extension/pypy/interpreter/test/test_gateway.py pypy/branch/cpython-extension/pypy/interpreter/test/test_objspace.py pypy/branch/cpython-extension/pypy/interpreter/typedef.py pypy/branch/cpython-extension/pypy/jit/backend/cli/test/conftest.py (props changed) pypy/branch/cpython-extension/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/branch/cpython-extension/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/branch/cpython-extension/pypy/jit/metainterp/codewriter.py pypy/branch/cpython-extension/pypy/jit/metainterp/logger.py (props changed) pypy/branch/cpython-extension/pypy/jit/metainterp/optimizefindnode.py pypy/branch/cpython-extension/pypy/jit/metainterp/optimizeopt.py pypy/branch/cpython-extension/pypy/jit/metainterp/pyjitpl.py pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/cpython-extension/pypy/module/__builtin__/app_inspect.py pypy/branch/cpython-extension/pypy/module/_codecs/__init__.py pypy/branch/cpython-extension/pypy/module/_codecs/app_codecs.py pypy/branch/cpython-extension/pypy/module/_codecs/interp_codecs.py pypy/branch/cpython-extension/pypy/module/_file/interp_file.py pypy/branch/cpython-extension/pypy/module/_file/interp_stream.py pypy/branch/cpython-extension/pypy/module/_file/test/test_file.py pypy/branch/cpython-extension/pypy/module/_locale/__init__.py pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py pypy/branch/cpython-extension/pypy/module/_stackless/interp_coroutine.py pypy/branch/cpython-extension/pypy/module/bz2/interp_bz2.py pypy/branch/cpython-extension/pypy/module/exceptions/ (props changed) pypy/branch/cpython-extension/pypy/module/posix/interp_posix.py pypy/branch/cpython-extension/pypy/module/posix/test/test_posix2.py pypy/branch/cpython-extension/pypy/module/pypyjit/policy.py pypy/branch/cpython-extension/pypy/module/pypyjit/test/test_policy.py pypy/branch/cpython-extension/pypy/module/signal/interp_signal.py pypy/branch/cpython-extension/pypy/module/sys/__init__.py pypy/branch/cpython-extension/pypy/module/sys/app.py pypy/branch/cpython-extension/pypy/module/sys/interp_encoding.py pypy/branch/cpython-extension/pypy/module/sys/test/test_sysmodule.py pypy/branch/cpython-extension/pypy/module/termios/interp_termios.py pypy/branch/cpython-extension/pypy/objspace/flow/flowcontext.py pypy/branch/cpython-extension/pypy/objspace/flow/model.py pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py pypy/branch/cpython-extension/pypy/objspace/flow/operation.py pypy/branch/cpython-extension/pypy/objspace/flow/specialcase.py pypy/branch/cpython-extension/pypy/objspace/flow/test/test_objspace.py pypy/branch/cpython-extension/pypy/objspace/std/basestringtype.py pypy/branch/cpython-extension/pypy/objspace/std/dictmultiobject.py pypy/branch/cpython-extension/pypy/objspace/std/formatting.py pypy/branch/cpython-extension/pypy/objspace/std/frame.py pypy/branch/cpython-extension/pypy/objspace/std/intobject.py pypy/branch/cpython-extension/pypy/objspace/std/objspace.py pypy/branch/cpython-extension/pypy/objspace/std/smallintobject.py pypy/branch/cpython-extension/pypy/objspace/std/stringobject.py pypy/branch/cpython-extension/pypy/objspace/std/test/test_obj.py pypy/branch/cpython-extension/pypy/objspace/std/test/test_setobject.py (props changed) pypy/branch/cpython-extension/pypy/objspace/std/test/test_stringformat.py pypy/branch/cpython-extension/pypy/objspace/std/unicodetype.py pypy/branch/cpython-extension/pypy/rlib/nonconst.py pypy/branch/cpython-extension/pypy/rlib/rsre/rsre_char.py pypy/branch/cpython-extension/pypy/rlib/test/test_rcoroutine.py (props changed) pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/cpython-extension/pypy/rpython/memory/gctransform/framework.py pypy/branch/cpython-extension/pypy/rpython/rint.py pypy/branch/cpython-extension/pypy/rpython/test/test_llann.py pypy/branch/cpython-extension/pypy/rpython/test/test_rlist.py pypy/branch/cpython-extension/pypy/rpython/test/test_rpbc.py pypy/branch/cpython-extension/pypy/tool/pytest/appsupport.py pypy/branch/cpython-extension/pypy/tool/sourcetools.py pypy/branch/cpython-extension/pypy/tool/stdlib_opcode.py pypy/branch/cpython-extension/pypy/translator/c/database.py pypy/branch/cpython-extension/pypy/translator/c/gc.py pypy/branch/cpython-extension/pypy/translator/c/node.py pypy/branch/cpython-extension/pypy/translator/c/src/signals.h pypy/branch/cpython-extension/pypy/translator/c/test/test_refcount.py (props changed) pypy/branch/cpython-extension/pypy/translator/driver.py pypy/branch/cpython-extension/pypy/translator/geninterplevel.py pypy/branch/cpython-extension/pypy/translator/goal/nanos.py pypy/branch/cpython-extension/pypy/translator/goal/targetpypystandalone.py pypy/branch/cpython-extension/pypy/translator/jvm/conftest.py pypy/branch/cpython-extension/pypy/translator/simplify.py pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py pypy/branch/cpython-extension/pypy/translator/tool/test/test_cbuild.py pypy/branch/cpython-extension/pypy/translator/translator.py Log: merge from trunk Modified: pypy/branch/cpython-extension/py/impl/code/_assertionold.py ============================================================================== --- pypy/branch/cpython-extension/py/impl/code/_assertionold.py (original) +++ pypy/branch/cpython-extension/py/impl/code/_assertionold.py Fri Apr 23 02:41:35 2010 @@ -447,7 +447,6 @@ def check(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) expr = parse(s, 'eval') @@ -518,7 +517,6 @@ def run(s, frame=None): if frame is None: - import sys frame = sys._getframe(1) frame = py.code.Frame(frame) module = Interpretable(parse(s, 'exec').node) Modified: pypy/branch/cpython-extension/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/cpython-extension/pypy/annotation/annrpython.py (original) +++ pypy/branch/cpython-extension/pypy/annotation/annrpython.py Fri Apr 23 02:41:35 2010 @@ -1,17 +1,18 @@ -from types import FunctionType +import sys +import types from pypy.tool.ansi_print import ansi_log, raise_nicer_exception -from pypy.annotation import model as annmodel from pypy.tool.pairtype import pair +from pypy.tool.error import (format_blocked_annotation_error, + format_someobject_error, AnnotatorError) +from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, + c_last_exception, checkgraph) +from pypy.translator import simplify, transform +from pypy.annotation import model as annmodel, signature from pypy.annotation.bookkeeper import Bookkeeper -from pypy.annotation import signature -from pypy.objspace.flow.model import Variable, Constant -from pypy.objspace.flow.model import FunctionGraph -from pypy.objspace.flow.model import c_last_exception, checkgraph import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) +log = py.log.Producer("annrpython") +py.log.setconsumer("annrpython", ansi_log) -from pypy.tool.error import format_blocked_annotation_error, format_someobject_error, AnnotatorError FAIL = object() @@ -84,7 +85,7 @@ def build_types(self, function, input_arg_types, complete_now=True): """Recursively build annotations about the specific entry point.""" - assert isinstance(function, FunctionType), "fix that!" + assert isinstance(function, types.FunctionType), "fix that!" from pypy.annotation.policy import AnnotatorPolicy policy = AnnotatorPolicy() @@ -429,10 +430,8 @@ def simplify(self, block_subset=None, extra_passes=None): # Generic simplifications - from pypy.translator import transform transform.transform_graph(self, block_subset=block_subset, extra_passes=extra_passes) - from pypy.translator import simplify if block_subset is None: graphs = self.translator.graphs else: @@ -534,7 +533,6 @@ except BlockedInference, e: if annmodel.DEBUG: - import sys self.why_not_annotated[block] = sys.exc_info() if (e.op is block.operations[-1] and @@ -611,12 +609,11 @@ candidates = [c for c in candidates if c not in covered] for link in exits: - import types in_except_block = False last_exception_var = link.last_exception # may be None for non-exception link last_exc_value_var = link.last_exc_value # may be None for non-exception link - + if isinstance(link.exitcase, (types.ClassType, type)) \ and issubclass(link.exitcase, py.builtin.BaseException): assert last_exception_var and last_exc_value_var Modified: pypy/branch/cpython-extension/pypy/config/translationoption.py ============================================================================== --- pypy/branch/cpython-extension/pypy/config/translationoption.py (original) +++ pypy/branch/cpython-extension/pypy/config/translationoption.py Fri Apr 23 02:41:35 2010 @@ -103,7 +103,7 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - suggests=[("translation.gc", "hybrid"), # or "boehm" + suggests=[("translation.gc", "hybrid"), ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), ChoiceOption("jit_backend", "choose the backend for the JIT", Modified: pypy/branch/cpython-extension/pypy/conftest.py ============================================================================== --- pypy/branch/cpython-extension/pypy/conftest.py (original) +++ pypy/branch/cpython-extension/pypy/conftest.py Fri Apr 23 02:41:35 2010 @@ -1,5 +1,4 @@ import py, sys, os -from py.impl.test.outcome import Failed from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport Modified: pypy/branch/cpython-extension/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/astcompiler/assemble.py Fri Apr 23 02:41:35 2010 @@ -400,7 +400,8 @@ self.first_lineno, lnotab, free_names, - cell_names) + cell_names, + self.compile_info.hidden_applevel) def _list_from_dict(d, offset=0): Modified: pypy/branch/cpython-extension/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/astcompiler/test/test_compiler.py Fri Apr 23 02:41:35 2010 @@ -22,11 +22,15 @@ """ def run(self, source): + import sys source = str(py.code.Source(source)) space = self.space code = compile_with_astcompiler(source, 'exec', space) - print - code.dump() + # 2.7 bytecode is too different, the standard `dis` module crashes + # when trying to display pypy (2.5-like) bytecode. + if sys.version_info < (2, 7): + print + code.dump() w_dict = space.newdict() code.exec_code(space, w_dict, w_dict) return w_dict @@ -39,7 +43,12 @@ pyco_expr = PyCode._from_code(space, co_expr) w_res = pyco_expr.exec_code(space, w_dict, w_dict) res = space.str_w(space.repr(w_res)) - assert res == repr(expected) + if not isinstance(expected, float): + assert res == repr(expected) + else: + # Float representation can vary a bit between interpreter + # versions, compare the numbers instead. + assert eval(res) == expected def simple_test(self, source, evalexpr, expected): w_g = self.run(source) Modified: pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py Fri Apr 23 02:41:35 2010 @@ -13,7 +13,7 @@ from pypy.rlib.timer import DummyTimer, Timer from pypy.rlib.rarithmetic import r_uint from pypy.rlib import jit -import os, sys +import os, sys, py __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -42,7 +42,7 @@ def setdictvalue(self, space, attr, w_value, shadows_type=True): w_dict = self.getdict() if w_dict is not None: - space.set_str_keyed_item(w_dict, attr, w_value, shadows_type) + space.setitem_str(w_dict, attr, w_value, shadows_type) return True return False @@ -209,12 +209,6 @@ def ready(self, result): pass -class UnpackValueError(ValueError): - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - class DescrMismatch(Exception): pass @@ -257,8 +251,10 @@ self.actionflag.register_action(self.user_del_action) self.actionflag.register_action(self.frame_trace_action) - from pypy.interpreter.pyframe import PyFrame - self.FrameClass = PyFrame # can be overridden to a subclass + from pypy.interpreter.pycode import cpython_magic, default_magic + self.our_magic = default_magic + self.host_magic = cpython_magic + # can be overridden to a subclass if self.config.objspace.logbytecodes: self.bytecodecounts = [0] * 256 @@ -628,7 +624,7 @@ """shortcut for space.int_w(space.hash(w_obj))""" return self.int_w(self.hash(w_obj)) - def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + def setitem_str(self, w_obj, key, w_value, shadows_type=True): return self.setitem(w_obj, self.wrap(key), w_value) def finditem_str(self, w_obj, key): @@ -721,7 +717,8 @@ raise break # done if expected_length != -1 and len(items) == expected_length: - raise UnpackValueError("too many values to unpack") + raise OperationError(space.w_ValueError, + space.wrap("too many values to unpack")) items.append(w_item) if expected_length != -1 and len(items) < expected_length: i = len(items) @@ -729,8 +726,9 @@ plural = "" else: plural = "s" - raise UnpackValueError("need more than %d value%s to unpack" % - (i, plural)) + raise OperationError(space.w_ValueError, + space.wrap("need more than %d value%s to unpack" % + (i, plural))) return items def fixedview(self, w_iterable, expected_length=-1): @@ -877,6 +875,9 @@ w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + def isinstance_w(self, w_obj, w_type): + return self.is_true(self.isinstance(w_obj, w_type)) + # The code below only works # for the simple case (new-style instance). # These methods are patched with the full logic by the __builtin__ @@ -928,23 +929,30 @@ import types from pypy.interpreter.pycode import PyCode if isinstance(expression, str): - expression = compile(expression, '?', 'eval') + compiler = self.createcompiler() + expression = compiler.compile(expression, '?', 'eval', 0, + hidden_applevel=hidden_applevel) if isinstance(expression, types.CodeType): - expression = PyCode._from_code(self, expression, - hidden_applevel=hidden_applevel) + # XXX only used by appsupport + expression = PyCode._from_code(self, expression) if not isinstance(expression, PyCode): raise TypeError, 'space.eval(): expected a string, code or PyCode object' return expression.exec_code(self, w_globals, w_locals) - def exec_(self, statement, w_globals, w_locals, hidden_applevel=False): + def exec_(self, statement, w_globals, w_locals, hidden_applevel=False, + filename=None): "NOT_RPYTHON: For internal debugging." import types + if filename is None: + filename = '?' from pypy.interpreter.pycode import PyCode if isinstance(statement, str): - statement = compile(statement, '?', 'exec') + compiler = self.createcompiler() + statement = compiler.compile(statement, filename, 'exec', 0, + hidden_applevel=hidden_applevel) if isinstance(statement, types.CodeType): - statement = PyCode._from_code(self, statement, - hidden_applevel=hidden_applevel) + # XXX only used by appsupport + statement = PyCode._from_code(self, statement) if not isinstance(statement, PyCode): raise TypeError, 'space.exec_(): expected a string, code or PyCode object' w_key = self.wrap('__builtins__') @@ -1095,6 +1103,17 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) + def path_w(space, w_obj): + """ Like str_w, but if the object is unicode, encode it using + filesystemencoding + """ + filesystemencoding = space.sys.filesystemencoding + if (filesystemencoding and + space.is_true(space.isinstance(w_obj, space.w_unicode))): + w_obj = space.call_method(w_obj, "encode", + space.wrap(filesystemencoding)) + return space.str_w(w_obj) + def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). @@ -1162,7 +1181,7 @@ assert source.startswith('('), "incorrect header in:\n%s" % (source,) source = py.code.Source("def anonymous%s\n" % source) w_glob = space.newdict() - space.exec_(source.compile(), w_glob, w_glob) + space.exec_(str(source), w_glob, w_glob) return space.getitem(w_glob, space.wrap('anonymous')) class DummyLock(object): Modified: pypy/branch/cpython-extension/pypy/interpreter/function.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/function.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/function.py Fri Apr 23 02:41:35 2010 @@ -235,15 +235,17 @@ def _freeze_(self): from pypy.interpreter.gateway import BuiltinCode if isinstance(self.code, BuiltinCode): - identifier = self.code.identifier - if Function._all.get(identifier, self) is not self: - print "builtin code identifier %s used twice: %s and %s" % ( - identifier, self, Function._all[identifier]) # we have been seen by other means so rtyping should not choke # on us - Function._all[identifier] = self + identifier = self.code.identifier + assert Function._all.get(identifier, self) is self, ("duplicate " + "function ids") + self.add_to_table() return False + def add_to_table(self): + Function._all[self.code.identifier] = self + def find(identifier): return Function._all[identifier] find = staticmethod(find) Modified: pypy/branch/cpython-extension/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/gateway.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/gateway.py Fri Apr 23 02:41:35 2010 @@ -137,6 +137,9 @@ def visit_c_nonnegint(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_path(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit__Wrappable(self, el, app_sig): name = el.__name__ argname = self.orig_arg() @@ -238,6 +241,9 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) + def visit_path(self, typ): + self.run_args.append("space.path_w(%s)" % (self.scopenext(),)) + def visit_nonnegint(self, typ): self.run_args.append("space.nonnegint_w(%s)" % (self.scopenext(),)) @@ -365,6 +371,9 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) + def visit_path(self, typ): + self.unwrap.append("space.path_w(%s)" % (self.nextarg(),)) + def visit_nonnegint(self, typ): self.unwrap.append("space.nonnegint_w(%s)" % (self.nextarg(),)) @@ -792,8 +801,8 @@ defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) - if not space.config.translating: # for tests and py.py - fn._freeze_() + if not space.config.translating: + fn.add_to_table() if gateway.as_classmethod: fn = ClassMethod(space.wrap(fn)) return fn @@ -812,6 +821,9 @@ # and now for something completely different ... # +class MyStr(str): + pass + class ApplevelClass: """NOT_RPYTHON A container for app-level source code that should be executed @@ -821,12 +833,14 @@ hidden_applevel = True - def __init__(self, source, filename = None, modname = '__builtin__'): + def __init__(self, source, filename=None, modname='__builtin__'): + # HAAACK (but a good one) + if filename is None: + f = sys._getframe(1) + filename = MyStr('<%s:%d>' % (f.f_code.co_filename, f.f_lineno)) + filename.__source__ = py.code.Source(source) self.filename = filename - if self.filename is None: - self.code = py.code.Source(source).compile() - else: - self.code = NiceCompile(self.filename)(source) + self.source = str(py.code.Source(source).deindent()) self.modname = modname # look at the first three lines for a NOT_RPYTHON tag first = "\n".join(source.split("\n", 3)[:3]) @@ -907,11 +921,11 @@ def build_applevel_dict(self, space): "NOT_RPYTHON" - from pypy.interpreter.pycode import PyCode w_glob = space.newdict(module=True) space.setitem(w_glob, space.wrap('__name__'), space.wrap(self.modname)) - space.exec_(self.code, w_glob, w_glob, - hidden_applevel=self.hidden_applevel) + space.exec_(self.source, w_glob, w_glob, + hidden_applevel=self.hidden_applevel, + filename=self.filename) return w_glob # __________ geninterplevel version __________ @@ -931,7 +945,7 @@ from pypy.translator.geninterplevel import translate_as_module import marshal scramble = md5(cls.seed) - scramble.update(marshal.dumps(self.code)) + scramble.update(marshal.dumps(self.source)) key = scramble.hexdigest() initfunc = cls.known_code.get(key) if not initfunc: @@ -952,7 +966,7 @@ if not initfunc: # build it and put it into a file initfunc, newsrc = translate_as_module( - self.code, self.filename, self.modname) + self.source, self.filename, self.modname) fname = cls.cache_path.join(name+".py").strpath f = file(get_tmp_file_name(fname), "w") print >> f, """\ Modified: pypy/branch/cpython-extension/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pycode.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pycode.py Fri Apr 23 02:41:35 2010 @@ -117,6 +117,10 @@ self._compute_flatcall() + def _freeze_(self): + if self.magic == cpython_magic: + raise Exception("CPython host codes should not be rendered") + return False def _init_flags(self): co_code = self.co_code Modified: pypy/branch/cpython-extension/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pycompiler.py Fri Apr 23 02:41:35 2010 @@ -151,8 +151,9 @@ e.wrap_info(space)) return mod - def compile(self, source, filename, mode, flags): + def compile(self, source, filename, mode, flags, hidden_applevel=False): from pypy.interpreter.pyparser.pyparse import CompileInfo - info = CompileInfo(filename, mode, flags) + info = CompileInfo(filename, mode, flags, + hidden_applevel=hidden_applevel) mod = self._compile_to_ast(source, info) return self._compile_ast(mod, info) Modified: pypy/branch/cpython-extension/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyframe.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyframe.py Fri Apr 23 02:41:35 2010 @@ -10,7 +10,7 @@ from pypy.rlib.objectmodel import we_are_translated, instantiate from pypy.rlib.jit import hint from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib import jit +from pypy.rlib import jit, rstack from pypy.tool import stdlib_opcode # Define some opcodes used @@ -132,10 +132,9 @@ def execute_frame(self): """Execute this frame. Main entry point to the interpreter.""" - from pypy.rlib import rstack # the following 'assert' is an annotation hint: it hides from # the annotator all methods that are defined in PyFrame but - # overridden in the FrameClass subclass of PyFrame. + # overridden in the {,Host}FrameClass subclasses of PyFrame. assert isinstance(self, self.space.FrameClass) executioncontext = self.space.getexecutioncontext() executioncontext.enter(self) @@ -238,6 +237,8 @@ self.pushvalue(w_value) def peekvalue(self, index_from_top=0): + # NOTE: top of the stack is peekvalue(0). + # Contrast this with CPython where it's PEEK(-1). index_from_top = hint(index_from_top, promote=True) index = self.valuestackdepth + ~index_from_top assert index >= 0, "peek past the bottom of the stack" Modified: pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyopcode.py Fri Apr 23 02:41:35 2010 @@ -6,17 +6,17 @@ import sys from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.baseobjspace import UnpackValueError, Wrappable +from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter import gateway, function, eval, pyframe, pytraceback from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib import jit, rstackovf +from pypy.rlib import jit, rstackovf, rstack from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.tool.stdlib_opcode import (opcodedesc, HAVE_ARGUMENT, - unrolling_opcode_descs, - opcode_method_names) +from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec, + unrolling_all_opcode_descs, opmap, + host_opmap) def unaryoperation(operationname): """NOT_RPYTHON""" @@ -66,12 +66,16 @@ # for logbytecode: last_opcode = -1 + bytecode_spec = bytecode_spec + opcode_method_names = bytecode_spec.method_names + opcodedesc = bytecode_spec.opcodedesc + opdescmap = bytecode_spec.opdescmap + HAVE_ARGUMENT = bytecode_spec.HAVE_ARGUMENT + ### opcode dispatch ### def dispatch(self, pycode, next_instr, ec): # For the sequel, force 'next_instr' to be unsigned for performance - from pypy.rlib import rstack # for resume points - next_instr = r_uint(next_instr) co_code = pycode.co_code @@ -84,8 +88,6 @@ return self.popvalue() def handle_bytecode(self, co_code, next_instr, ec): - from pypy.rlib import rstack # for resume points - try: next_instr = self.dispatch_bytecode(co_code, next_instr, ec) rstack.resume_point("handle_bytecode", self, co_code, ec, @@ -180,7 +182,7 @@ probs[opcode] = probs.get(opcode, 0) + 1 self.last_opcode = opcode - if opcode >= HAVE_ARGUMENT: + if opcode >= self.HAVE_ARGUMENT: lo = ord(co_code[next_instr]) hi = ord(co_code[next_instr+1]) next_instr += 2 @@ -188,16 +190,16 @@ else: oparg = 0 - while opcode == opcodedesc.EXTENDED_ARG.index: + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) - if opcode < HAVE_ARGUMENT: + if opcode < self.HAVE_ARGUMENT: raise BytecodeCorruption lo = ord(co_code[next_instr+1]) hi = ord(co_code[next_instr+2]) next_instr += 3 oparg = (oparg << 16) | (hi << 8) | lo - if opcode == opcodedesc.RETURN_VALUE.index: + if opcode == self.opcodedesc.RETURN_VALUE.index: w_returnvalue = self.popvalue() block = self.unrollstack(SReturnValue.kind) if block is None: @@ -208,11 +210,11 @@ next_instr = block.handle(self, unroller) return next_instr # now inside a 'finally' block - if opcode == opcodedesc.YIELD_VALUE.index: + if opcode == self.opcodedesc.YIELD_VALUE.index: #self.last_instr = intmask(next_instr - 1) XXX clean up! raise Yield - if opcode == opcodedesc.END_FINALLY.index: + if opcode == self.opcodedesc.END_FINALLY.index: unroller = self.end_finally() if isinstance(unroller, SuspendedUnroller): # go on unrolling the stack @@ -225,25 +227,28 @@ next_instr = block.handle(self, unroller) return next_instr - if opcode == opcodedesc.JUMP_ABSOLUTE.index: + if opcode == self.opcodedesc.JUMP_ABSOLUTE.index: return self.jump_absolute(oparg, next_instr, ec) if we_are_translated(): - from pypy.rlib import rstack # for resume points - - for opdesc in unrolling_opcode_descs: + for opdesc in unrolling_all_opcode_descs: # static checks to skip this whole case if necessary + if opdesc.bytecode_spec is not self.bytecode_spec: + continue if not opdesc.is_enabled(space): continue - if not hasattr(pyframe.PyFrame, opdesc.methodname): - continue # e.g. for JUMP_ABSOLUTE, implemented above + if opdesc.methodname in ( + 'EXTENDED_ARG', 'RETURN_VALUE', 'YIELD_VALUE', + 'END_FINALLY', 'JUMP_ABSOLUTE'): + continue # opcodes implemented above if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) res = meth(oparg, next_instr) - if opdesc.index == opcodedesc.CALL_FUNCTION.index: - rstack.resume_point("dispatch_call", self, co_code, next_instr, ec) + if opdesc.index == self.opcodedesc.CALL_FUNCTION.index: + rstack.resume_point("dispatch_call", self, co_code, + next_instr, ec) # !! warning, for the annotator the next line is not # comparing an int and None - you can't do that. # Instead, it's constant-folded to either True or False @@ -254,8 +259,27 @@ self.MISSING_OPCODE(oparg, next_instr) else: # when we are not translated, a list lookup is much faster - methodname = opcode_method_names[opcode] - res = getattr(self, methodname)(oparg, next_instr) + methodname = self.opcode_method_names[opcode] + try: + meth = getattr(self, methodname) + except AttributeError: + raise BytecodeCorruption("unimplemented opcode, ofs=%d, " + "code=%d, name=%s" % + (self.last_instr, opcode, + methodname)) + try: + res = meth(oparg, next_instr) + except Exception: + if 0: + import dis, sys + print "*** %s at offset %d (%s)" % (sys.exc_info()[0], + self.last_instr, + methodname) + try: + dis.dis(co_code) + except: + pass + raise if res is not None: next_instr = res @@ -536,9 +560,8 @@ # common case raise operror else: - from pypy.interpreter.pytraceback import check_traceback msg = "raise: arg 3 must be a traceback or None" - tb = check_traceback(space, w_traceback, msg) + tb = pytraceback.check_traceback(space, w_traceback, msg) operror.application_traceback = tb # special 3-arguments raise, no new traceback obj will be attached raise RaiseWithExplicitTraceback(operror) @@ -599,7 +622,7 @@ def STORE_NAME(self, varindex, next_instr): varname = self.getname_u(varindex) w_newvalue = self.popvalue() - self.space.set_str_keyed_item(self.w_locals, varname, w_newvalue) + self.space.setitem_str(self.w_locals, varname, w_newvalue) def DELETE_NAME(self, varindex, next_instr): w_varname = self.getname_w(varindex) @@ -615,11 +638,7 @@ def UNPACK_SEQUENCE(self, itemcount, next_instr): w_iterable = self.popvalue() - try: - items = self.space.fixedview(w_iterable, itemcount) - except UnpackValueError, e: - w_msg = self.space.wrap(e.msg) - raise OperationError(self.space.w_ValueError, w_msg) + items = self.space.fixedview(w_iterable, itemcount) self.pushrevvalues(itemcount, items) def STORE_ATTR(self, nameindex, next_instr): @@ -638,7 +657,7 @@ def STORE_GLOBAL(self, nameindex, next_instr): varname = self.getname_u(nameindex) w_newvalue = self.popvalue() - self.space.set_str_keyed_item(self.w_globals, varname, w_newvalue) + self.space.setitem_str(self.w_globals, varname, w_newvalue) def DELETE_GLOBAL(self, nameindex, next_instr): w_varname = self.getname_w(nameindex) @@ -690,26 +709,6 @@ w_list = self.space.newlist(items) self.pushvalue(w_list) - def BUILD_MAP(self, itemcount, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - # We could pre-allocate a dict here - # but for the moment this code is not translated. - pass - else: - if itemcount != 0: - raise BytecodeCorruption - w_dict = self.space.newdict() - self.pushvalue(w_dict) - - def STORE_MAP(self, zero, next_instr): - if not we_are_translated() and sys.version_info >= (2, 6): - w_key = self.popvalue() - w_value = self.popvalue() - w_dict = self.peekvalue() - self.space.setitem(w_dict, w_key, w_value) - else: - raise BytecodeCorruption - def LOAD_ATTR(self, nameindex, next_instr): "obj.attributename" w_attributename = self.getname_w(nameindex) @@ -887,7 +886,6 @@ @jit.unroll_safe def call_function(self, oparg, w_star=None, w_starstar=None): - from pypy.rlib import rstack # for resume points from pypy.interpreter.function import is_builtin_code n_arguments = oparg & 0xff @@ -920,8 +918,6 @@ self.pushvalue(w_result) def CALL_FUNCTION(self, oparg, next_instr): - from pypy.rlib import rstack # for resume points - # XXX start of hack for performance if (oparg >> 8) & 0xff == 0: # Only positional arguments @@ -1011,14 +1007,22 @@ self.pushvalue(w_result) def MISSING_OPCODE(self, oparg, next_instr): - ofs = next_instr - 1 + ofs = self.last_instr c = self.pycode.co_code[ofs] name = self.pycode.co_name raise BytecodeCorruption("unknown opcode, ofs=%d, code=%d, name=%s" % (ofs, ord(c), name) ) STOP_CODE = MISSING_OPCODE + + def BUILD_MAP(self, itemcount, next_instr): + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + def STORE_MAP(self, zero, next_instr): + raise BytecodeCorruption ### ____________________________________________________________ ### Modified: pypy/branch/cpython-extension/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyparser/metaparser.py Fri Apr 23 02:41:35 2010 @@ -128,6 +128,7 @@ def build_grammar(self, grammar_cls): gram = grammar_cls() + gram.start = self.start_symbol names = self.dfas.keys() names.sort() names.remove(self.start_symbol) @@ -146,8 +147,8 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - our_id = gram.symbol_ids[name] - gram.dfas[our_id] = (states, self.make_first(gram, name)) + gram.dfas.append((states, self.make_first(gram, name))) + assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -229,7 +230,8 @@ for label, their_first in overlap_check.iteritems(): for sub_label in their_first: if sub_label in inverse: - raise PgenError("ambiguous symbol %s" % (symbol,)) + raise PgenError("ambiguous symbol with label %s" + % (label,)) inverse[sub_label] = label self.first[name] = all_labels return all_labels Modified: pypy/branch/cpython-extension/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyparser/parser.py Fri Apr 23 02:41:35 2010 @@ -16,9 +16,10 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} - self.dfas = {} + self.dfas = [] self.labels = [0] self.token_ids = {} + self.start = -1 def shared_copy(self): new = self.__class__() @@ -98,7 +99,7 @@ self.root = None current_node = Node(start, None, [], 0, 0) self.stack = [] - self.stack.append((self.grammar.dfas[start], 0, current_node)) + self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) @@ -124,7 +125,7 @@ state = dfa[0][state_index] return False elif sym_id >= 256: - sub_node_dfa = self.grammar.dfas[sym_id] + sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. if label_index in sub_node_dfa[1]: self.push(sub_node_dfa, next_state, sym_id, lineno, Modified: pypy/branch/cpython-extension/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyparser/pyparse.py Fri Apr 23 02:41:35 2010 @@ -65,14 +65,18 @@ * encoding: The source encoding. * last_future_import: The line number and offset of the last __future__ import. + * hidden_applevel: Will this code unit and sub units be hidden at the + applevel? """ - def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0)): + def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0), + hidden_applevel=False): self.filename = filename self.mode = mode self.encoding = None self.flags = flags self.last_future_import = future_pos + self.hidden_applevel = hidden_applevel _targets = { Modified: pypy/branch/cpython-extension/pypy/interpreter/pyparser/test/test_metaparser.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/pyparser/test/test_metaparser.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/pyparser/test/test_metaparser.py Fri Apr 23 02:41:35 2010 @@ -33,9 +33,8 @@ g = self.gram_for("eval: NAME\n") assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] - assert eval_sym in g.dfas assert g.start == eval_sym - states, first = g.dfas[eval_sym] + states, first = g.dfas[eval_sym - 256] assert states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 @@ -52,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"]][0] + states = g.dfas[g.symbol_ids["foo"] - 256][0] last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] Modified: pypy/branch/cpython-extension/pypy/interpreter/test/test_appinterp.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/test/test_appinterp.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/test/test_appinterp.py Fri Apr 23 02:41:35 2010 @@ -1,6 +1,7 @@ import py from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp, applevelinterp_temp +from pypy.interpreter.error import OperationError def test_execwith_novars(space): val = space.appexec([], """ @@ -18,11 +19,11 @@ assert space.eq_w(val, space.wrap(42)) def test_execwith_compile_error(space): - excinfo = py.test.raises(SyntaxError, space.appexec, [], """ + excinfo = py.test.raises(OperationError, space.appexec, [], """ (): y y """) - assert str(excinfo.value).find('y y') != -1 + assert str(excinfo.value.errorstr(space)).find('y y') != -1 def test_simple_applevel(space): app = appdef("""app(x,y): Modified: pypy/branch/cpython-extension/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/test/test_compiler.py Fri Apr 23 02:41:35 2010 @@ -54,6 +54,14 @@ space.raises_w(space.w_SyntaxError, self.compiler.compile_command, ')', '?', mode, 0) + def test_hidden_applevel(self): + code = self.compiler.compile("def f(x): pass", "", "exec", 0, + True) + assert code.hidden_applevel + for w_const in code.co_consts_w: + if isinstance(w_const, PyCode): + assert code.hidden_applevel + def test_indentation_error(self): space = self.space space.raises_w(space.w_SyntaxError, self.compiler.compile_command, Modified: pypy/branch/cpython-extension/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/test/test_gateway.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,6 @@ + +# -*- coding: utf-8 -*- + from pypy.conftest import gettestobjspace from pypy.interpreter import gateway from pypy.interpreter import argument @@ -451,6 +454,16 @@ assert len(l) == 1 assert space.eq_w(l[0], w("foo")) + def test_interp2app_unwrap_spec_path(self, monkeypatch): + space = self.space + def g(space, p): + return p + + app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'path']) + w_app_g = space.wrap(app_g) + monkeypatch.setattr(space.sys, "filesystemencoding", "utf-8") + w_res = space.call_function(w_app_g, space.wrap(u"?")) + def test_interp2app_classmethod(self): space = self.space w = space.wrap Modified: pypy/branch/cpython-extension/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/test/test_objspace.py Fri Apr 23 02:41:35 2010 @@ -12,7 +12,18 @@ INT32_MAX = 2147483648 -class TestObjSpace: +class TestObjSpace: + + def test_isinstance(self): + space = self.space + w_i = space.wrap(4) + w_result = space.isinstance(w_i, space.w_int) + assert space.is_true(w_result) + assert space.isinstance_w(w_i, space.w_int) + w_result = space.isinstance(w_i, space.w_str) + assert not space.is_true(w_result) + assert not space.isinstance_w(w_i, space.w_str) + def test_newlist(self): w = self.space.wrap l = range(10) @@ -50,31 +61,40 @@ assert self.space.newbool(1) == self.space.w_True def test_unpackiterable(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newlist(l) - assert self.space.unpackiterable(w_l) == l - assert self.space.unpackiterable(w_l, 4) == l - raises(ValueError, self.space.unpackiterable, w_l, 3) - raises(ValueError, self.space.unpackiterable, w_l, 5) + w_l = space.newlist(l) + assert space.unpackiterable(w_l) == l + assert space.unpackiterable(w_l, 4) == l + err = raises(OperationError, space.unpackiterable, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.unpackiterable, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_fixedview(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newtuple(l) - assert self.space.fixedview(w_l) == l - assert self.space.fixedview(w_l, 4) == l - raises(ValueError, self.space.fixedview, w_l, 3) - raises(ValueError, self.space.fixedview, w_l, 5) + w_l = space.newtuple(l) + assert space.fixedview(w_l) == l + assert space.fixedview(w_l, 4) == l + err = raises(OperationError, space.fixedview, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.fixedview, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_listview(self): - w = self.space.wrap + space = self.space + w = space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newtuple(l) - assert self.space.listview(w_l) == l - assert self.space.listview(w_l, 4) == l - raises(ValueError, self.space.listview, w_l, 3) - raises(ValueError, self.space.listview, w_l, 5) + w_l = space.newtuple(l) + assert space.listview(w_l) == l + assert space.listview(w_l, 4) == l + err = raises(OperationError, space.listview, w_l, 3) + assert err.value.match(space, space.w_ValueError) + err = raises(OperationError, space.listview, w_l, 5) + assert err.value.match(space, space.w_ValueError) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, Modified: pypy/branch/cpython-extension/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/typedef.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/typedef.py Fri Apr 23 02:41:35 2010 @@ -904,6 +904,3 @@ SuspendedUnroller.typedef = TypeDef("SuspendedUnroller") SuspendedUnroller.typedef.acceptable_as_base_class = False - - -interptypes = [ val.typedef for name,val in globals().items() if hasattr(val,'__bases__') and hasattr(val,'typedef') ] Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/codewriter.py Fri Apr 23 02:41:35 2010 @@ -394,7 +394,6 @@ self.assembler = [] self.constants = [] self.positions = {} - self.blocks = {} self.seen_blocks = {} self.dont_minimize_variables = 0 self.pending_exception_handlers = [] Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/optimizefindnode.py Fri Apr 23 02:41:35 2010 @@ -63,13 +63,14 @@ # invariant: if escaped=True, then dependencies is None if not self.escaped: self.escaped = True + self.unique = UNIQUE_NO + # ^^^ always set unique to UNIQUE_NO when we set escaped to True. + # See for example test_find_nodes_store_into_loop_constant_2. if self.dependencies is not None: deps = self.dependencies self.dependencies = None for box in deps: box.mark_escaped() - # see test_find_nodes_store_into_loop_constant_1 for this: - box.unique = UNIQUE_NO def set_unique_nodes(self): if self.fromstart: @@ -342,12 +343,16 @@ # computed by NodeFinder.find_nodes(). op = loop.operations[-1] assert op.opnum == rop.JUMP - specnodes = [] assert len(self.inputnodes) == len(op.args) - for i in range(len(op.args)): - inputnode = self.inputnodes[i] - exitnode = self.getnode(op.args[i]) - specnodes.append(self.intersect(inputnode, exitnode)) + while True: + self.restart_needed = False + specnodes = [] + for i in range(len(op.args)): + inputnode = self.inputnodes[i] + exitnode = self.getnode(op.args[i]) + specnodes.append(self.intersect(inputnode, exitnode)) + if not self.restart_needed: + break loop.token.specnodes = specnodes def intersect(self, inputnode, exitnode): @@ -362,6 +367,15 @@ return prebuiltNotSpecNode unique = exitnode.unique if unique == UNIQUE_NO: + if inputnode is not self.node_fromstart: + # Mark the input node as escaped, and schedule a complete + # restart of intersect(). This is needed because there is + # an order dependency: calling inputnode.mark_escaped() + # might set the field exitnode.unique to UNIQUE_NO in some + # other node. If inputnode is node_fromstart, there is no + # problem (and it must not be mutated by mark_escaped() then). + inputnode.mark_escaped() + self.restart_needed = True return prebuiltNotSpecNode if unique == UNIQUE_INST: return self.intersect_instance(inputnode, exitnode) Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/optimizeopt.py Fri Apr 23 02:41:35 2010 @@ -216,17 +216,8 @@ self._fields[ofs] = fieldvalue def _really_force(self): - if self.source_op is None: - # this case should not occur; I only managed to get it once - # in pypy-c-jit and couldn't reproduce it. The point is - # that it relies on optimizefindnode.py computing exactly - # the right level of specialization, and it seems that there - # is still a corner case where it gets too specialized for - # optimizeopt.py. Let's not crash in release-built - # pypy-c-jit's. XXX find out when - from pypy.rlib.debug import ll_assert - ll_assert(False, "_really_force: source_op is None") - raise InvalidLoop + assert self.source_op is not None + # ^^^ This case should not occur any more (see test_bug_3). # newoperations = self.optimizer.newoperations newoperations.append(self.source_op) Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/pyjitpl.py Fri Apr 23 02:41:35 2010 @@ -1933,11 +1933,6 @@ # boxes, in whichever direction is appropriate if expect_virtualizable: self.virtualizable_boxes = virtualizable_boxes - if self._already_allocated_resume_virtuals is not None: - # resuming from a ResumeGuardForcedDescr: load the new values - # currently stored on the virtualizable fields - self.load_fields_from_virtualizable() - return # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token # is and stays 0. Note the call to reset_vable_token() in Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizefindnode.py Fri Apr 23 02:41:35 2010 @@ -301,8 +301,8 @@ boxes, getnode = self.find_nodes(ops, 'Virtual(node_vtable, nextdescr=Not)') assert not getnode(boxes.p0).escaped - assert not getnode(boxes.p1).escaped - assert not getnode(boxes.p2).escaped + assert getnode(boxes.p1).escaped + assert getnode(boxes.p2).escaped assert getnode(boxes.p0).fromstart assert getnode(boxes.p1).fromstart assert getnode(boxes.p2).fromstart Modified: pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/cpython-extension/pypy/jit/metainterp/test/test_optimizeopt.py Fri Apr 23 02:41:35 2010 @@ -1740,7 +1740,6 @@ self.optimize_loop(ops, 'Not, VArray(arraydescr2, Not)', expected) - @py.test.mark.xfail def test_bug_3(self): ops = """ [p1] @@ -1761,7 +1760,49 @@ setfield_gc(p1a, p3a, descr=otherdescr) jump(p1a) """ - self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Virtual(node_vtable), otherdescr=Not)', None) + expected = """ + [p2, p3] + guard_class(p2, ConstClass(node_vtable)) [] + guard_class(p3, ConstClass(node_vtable)) [] + setfield_gc(p3, p2, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + p2a = new_with_vtable(ConstClass(node_vtable)) + jump(p2a, p3a) + """ + self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) + + def test_bug_3bis(self): + ops = """ + [p1] + guard_nonnull(p1) [] + guard_class(p1, ConstClass(node_vtable2)) [] + p2 = getfield_gc(p1, descr=nextdescr) + guard_nonnull(12) [] + guard_class(p2, ConstClass(node_vtable)) [] + p3 = getfield_gc(p1, descr=otherdescr) + guard_nonnull(12) [] + guard_class(p3, ConstClass(node_vtable)) [] + p1a = new_with_vtable(ConstClass(node_vtable2)) + p2a = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p3, p2a, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + setfield_gc(p1a, p2a, descr=nextdescr) + setfield_gc(p1a, p3a, descr=otherdescr) + jump(p1a) + """ + expected = """ + [p2, p3] + guard_class(p2, ConstClass(node_vtable)) [] + guard_class(p3, ConstClass(node_vtable)) [] + p2a = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p3, p2a, descr=otherdescr) + p3a = new_with_vtable(ConstClass(node_vtable)) + escape(p3a) + jump(p2a, p3a) + """ + self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) def test_invalid_loop_1(self): ops = """ Modified: pypy/branch/cpython-extension/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/branch/cpython-extension/pypy/module/__builtin__/app_inspect.py Fri Apr 23 02:41:35 2010 @@ -64,9 +64,7 @@ if isinstance(obj, types.ModuleType): try: - result = obj.__dict__.keys() - if not isinstance(result, list): - raise TypeError("expected __dict__.keys() to be a list") + result = list(obj.__dict__.keys()) result.sort() return result except AttributeError: Modified: pypy/branch/cpython-extension/pypy/module/_codecs/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_codecs/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/_codecs/__init__.py Fri Apr 23 02:41:35 2010 @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.rlib import runicode +from pypy.module._codecs import interp_codecs class Module(MixedModule): appleveldefs = { @@ -16,7 +17,6 @@ 'unicode_internal_encode' : 'app_codecs.unicode_internal_encode', 'utf_7_decode' : 'app_codecs.utf_7_decode', 'utf_7_encode' : 'app_codecs.utf_7_encode', - '_register_existing_errors': 'app_codecs._register_existing_errors', 'charmap_build' : 'app_codecs.charmap_build' } interpleveldefs = { @@ -57,9 +57,4 @@ MixedModule.__init__(self, space, *args) - def setup_after_space_initialization(self): - "NOT_RPYTHON" - self.space.appexec([], """(): - import _codecs - _codecs._register_existing_errors() - """) + interp_codecs.register_builtin_error_handlers(space) Modified: pypy/branch/cpython-extension/pypy/module/_codecs/app_codecs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_codecs/app_codecs.py (original) +++ pypy/branch/cpython-extension/pypy/module/_codecs/app_codecs.py Fri Apr 23 02:41:35 2010 @@ -1,7 +1,3 @@ -# NOT_RPYTHON -# Note: -# This *is* now explicitly RPython. -# Please make sure not to break this. """ @@ -218,79 +214,6 @@ res = ''.join(res) return res, len(res) -def check_exception(exc): - try: - delta = exc.end - exc.start - if delta < 0 or not isinstance(exc.object, (unicode, str)): - raise TypeError("wrong exception") - except AttributeError: - raise TypeError("wrong exception") - -def strict_errors(exc): - if isinstance(exc, Exception): - raise exc - else: - raise TypeError("codec must pass exception instance") - -def ignore_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'', exc.end - elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): - return u'', exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" - -def replace_errors(exc): - check_exception(exc) - if isinstance(exc, UnicodeEncodeError): - return u'?'*(exc.end-exc.start), exc.end - elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): - return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%exc) - -def xmlcharrefreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - res = [] - for ch in exc.object[exc.start:exc.end]: - res += '&#' - res += str(ord(ch)) - res += ';' - return u''.join(res), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - -def backslashreplace_errors(exc): - if isinstance(exc, UnicodeEncodeError): - p = [] - for c in exc.object[exc.start:exc.end]: - p += '\\' - oc = ord(c) - if (oc >= 0x00010000): - p += 'U' - p += "%.8x" % ord(c) - elif (oc >= 0x100): - p += 'u' - p += "%.4x" % ord(c) - else: - p += 'x' - p += "%.2x" % ord(c) - return u''.join(p), exc.end - else: - raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) - - -def _register_existing_errors(): - import _codecs - _codecs.register_error("strict", strict_errors) - _codecs.register_error("ignore", ignore_errors) - _codecs.register_error("replace", replace_errors) - _codecs.register_error("xmlcharrefreplace", xmlcharrefreplace_errors) - _codecs.register_error("backslashreplace", backslashreplace_errors) - # ---------------------------------------------------------------------- ##import sys @@ -356,6 +279,7 @@ bitsleft = 0 charsleft = 0 surrogate = 0 + startinpos = 0 p = [] errorHandler = None exc = None Modified: pypy/branch/cpython-extension/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/cpython-extension/pypy/module/_codecs/interp_codecs.py Fri Apr 23 02:41:35 2010 @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped +from pypy.interpreter.gateway import ObjSpace, NoneNotWrapped, applevel from pypy.interpreter.baseobjspace import W_Root from pypy.rlib.rstring import StringBuilder, UnicodeBuilder @@ -107,7 +107,80 @@ space.w_LookupError, "unknown encoding: %s", encoding) lookup_codec.unwrap_spec = [ObjSpace, str] - + +app_errors = applevel(""" +def check_exception(exc): + try: + delta = exc.end - exc.start + if delta < 0 or not isinstance(exc.object, (unicode, str)): + raise TypeError("wrong exception") + except AttributeError: + raise TypeError("wrong exception") + +def strict_errors(exc): + if isinstance(exc, Exception): + raise exc + else: + raise TypeError("codec must pass exception instance") + +def ignore_errors(exc): + check_exception(exc) + if isinstance(exc, UnicodeEncodeError): + return u'', exc.end + elif isinstance(exc, (UnicodeDecodeError, UnicodeTranslateError)): + return u'', exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%exc) + +Py_UNICODE_REPLACEMENT_CHARACTER = u"\ufffd" + +def replace_errors(exc): + check_exception(exc) + if isinstance(exc, UnicodeEncodeError): + return u'?'*(exc.end-exc.start), exc.end + elif isinstance(exc, (UnicodeTranslateError, UnicodeDecodeError)): + return Py_UNICODE_REPLACEMENT_CHARACTER*(exc.end-exc.start), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%exc) + +def xmlcharrefreplace_errors(exc): + if isinstance(exc, UnicodeEncodeError): + res = [] + for ch in exc.object[exc.start:exc.end]: + res += '&#' + res += str(ord(ch)) + res += ';' + return u''.join(res), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) + +def backslashreplace_errors(exc): + if isinstance(exc, UnicodeEncodeError): + p = [] + for c in exc.object[exc.start:exc.end]: + p += '\\\\' + oc = ord(c) + if (oc >= 0x00010000): + p += 'U' + p += "%.8x" % ord(c) + elif (oc >= 0x100): + p += 'u' + p += "%.4x" % ord(c) + else: + p += 'x' + p += "%.2x" % ord(c) + return u''.join(p), exc.end + else: + raise TypeError("don't know how to handle %.400s in error callback"%type(exc)) +""") + +def register_builtin_error_handlers(space): + state = space.fromcache(CodecState) + for error in ("strict", "ignore", "replace", "xmlcharrefreplace", + "backslashreplace"): + name = error + "_errors" + state.codec_error_registry[error] = app_errors.wget(space, name) + def lookup_error(space, errors): """lookup_error(errors) -> handler Modified: pypy/branch/cpython-extension/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_file/interp_file.py (original) +++ pypy/branch/cpython-extension/pypy/module/_file/interp_file.py Fri Apr 23 02:41:35 2010 @@ -24,9 +24,10 @@ # Default values until the file is successfully opened stream = None - name = "" + w_name = None mode = "" - encoding = None + softspace= 0 # Required according to file object docs + encoding = None # This is not used internally by file objects fd = -1 def __init__(self, space): @@ -38,12 +39,11 @@ self.clear_all_weakrefs() self.direct_close() - def fdopenstream(self, stream, fd, mode, w_name): + def fdopenstream(self, stream, fd, mode, w_name=None): self.fd = fd - self.w_name = w_name - self.softspace = 0 # Required according to file object docs - self.encoding = None # This is not used internally by file objects self.mode = mode + if w_name is not None: + self.w_name = w_name self.stream = stream if stream.flushable(): getopenstreams(self.space)[stream] = None @@ -83,10 +83,11 @@ def direct___init__(self, w_name, mode='r', buffering=-1): name = self.space.str_w(w_name) self.direct_close() + self.w_name = w_name self.check_mode_ok(mode) stream = streamio.open_file_as_stream(name, mode, buffering) fd = stream.try_to_find_file_descriptor() - self.fdopenstream(stream, fd, mode, w_name) + self.fdopenstream(stream, fd, mode) def direct___enter__(self): if self.stream is None: @@ -102,9 +103,10 @@ def direct_fdopen(self, fd, mode='r', buffering=-1): self.direct_close() + self.w_name = self.space.wrap('') self.check_mode_ok(mode) stream = streamio.fdopen_as_stream(fd, mode, buffering) - self.fdopenstream(stream, fd, mode, self.space.wrap('')) + self.fdopenstream(stream, fd, mode) def direct_close(self): space = self.space @@ -239,7 +241,7 @@ try: self.direct_fdopen(fd, mode, buffering) except StreamErrors, e: - raise wrap_streamerror(self.space, e) + raise wrap_streamerror(self.space, e, self.w_name) _exposed_method_names = [] @@ -275,7 +277,7 @@ try: result = self.direct_%(name)s(%(callsig)s) except StreamErrors, e: - raise wrap_streamerror(space, e) + raise wrap_streamerror(space, e, self.w_name) finally: self.unlock() return %(wrapresult)s @@ -387,20 +389,23 @@ head = "closed" else: head = "open" - if self.space.is_true(self.space.isinstance(self.w_name, - self.space.w_str)): - info = "%s file '%s', mode '%s'" % ( - head, - self.space.str_w(self.w_name), - self.mode) - else: - info = "%s file %s, mode '%s'" % ( - head, - self.space.str_w(self.space.repr(self.w_name)), - self.mode) + info = "%s file %s, mode '%s'" % ( + head, + self.getdisplayname(), + self.mode) return self.getrepr(self.space, info) file__repr__.unwrap_spec = ['self'] + def getdisplayname(self): + w_name = self.w_name + if w_name is None: + return '?' + elif self.space.is_true(self.space.isinstance(w_name, + self.space.w_str)): + return "'%s'" % self.space.str_w(w_name) + else: + return self.space.str_w(self.space.repr(w_name)) + def file_readinto(self, w_rwbuffer): """readinto() -> Undocumented. Don't use this; it may go away.""" # XXX not the most efficient solution as it doesn't avoid the copying Modified: pypy/branch/cpython-extension/pypy/module/_file/interp_stream.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_file/interp_stream.py (original) +++ pypy/branch/cpython-extension/pypy/module/_file/interp_stream.py Fri Apr 23 02:41:35 2010 @@ -10,16 +10,16 @@ import os -def wrap_streamerror(space, e): +def wrap_streamerror(space, e, w_filename=None): if isinstance(e, streamio.StreamError): return OperationError(space.w_ValueError, space.wrap(e.message)) elif isinstance(e, OSError): - return wrap_oserror_as_ioerror(space, e) + return wrap_oserror_as_ioerror(space, e, w_filename) else: return OperationError(space.w_IOError, space.w_None) -def wrap_oserror_as_ioerror(space, e): +def wrap_oserror_as_ioerror(space, e, w_filename=None): assert isinstance(e, OSError) errno = e.errno try: @@ -28,7 +28,8 @@ msg = 'error %d' % errno w_error = space.call_function(space.w_IOError, space.wrap(errno), - space.wrap(msg)) + space.wrap(msg), + w_filename) return OperationError(space.w_IOError, w_error) Modified: pypy/branch/cpython-extension/pypy/module/_file/test/test_file.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_file/test/test_file.py (original) +++ pypy/branch/cpython-extension/pypy/module/_file/test/test_file.py Fri Apr 23 02:41:35 2010 @@ -125,6 +125,13 @@ assert type(res) is str f.close() + def test_oserror_has_filename(self): + try: + f = self.file("file that is clearly not there") + except IOError, e: + assert e.filename == 'file that is clearly not there' + else: + raise Exception("did not raise") def test_readline_mixed_with_read(self): s = '''From MAILER-DAEMON Wed Jan 14 14:42:30 2009 Modified: pypy/branch/cpython-extension/pypy/module/_locale/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_locale/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/_locale/__init__.py Fri Apr 23 02:41:35 2010 @@ -1,27 +1,29 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module._locale import interp_locale +from pypy.rlib import rlocale import sys class Module(MixedModule): """Support for POSIX locales.""" interpleveldefs = { - 'setlocale': 'interp_locale.setlocale', - 'localeconv': 'interp_locale.localeconv', - 'strcoll': 'interp_locale.strcoll', - 'strxfrm': 'interp_locale.strxfrm', - } + 'setlocale': 'interp_locale.setlocale', + 'localeconv': 'interp_locale.localeconv', + 'strcoll': 'interp_locale.strcoll', + 'strxfrm': 'interp_locale.strxfrm', + 'Error': 'interp_locale.W_Error', + } if sys.platform == 'win32': interpleveldefs.update({ '_getdefaultlocale': 'interp_locale.getdefaultlocale', }) - if interp_locale.HAVE_LANGINFO: + if rlocale.HAVE_LANGINFO: interpleveldefs.update({ 'nl_langinfo': 'interp_locale.nl_langinfo', }) - if interp_locale.HAVE_LIBINTL: + if rlocale.HAVE_LIBINTL: interpleveldefs.update({ 'gettext': 'interp_locale.gettext', 'dgettext': 'interp_locale.dgettext', @@ -29,18 +31,16 @@ 'textdomain': 'interp_locale.textdomain', 'bindtextdomain': 'interp_locale.bindtextdomain', }) - if interp_locale.HAVE_BIND_TEXTDOMAIN_CODESET: + if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET: interpleveldefs.update({ 'bind_textdomain_codeset':'interp_locale.bind_textdomain_codeset', }) appleveldefs = { - 'Error': 'app_locale.Error', - '_fixup_ulcase': 'app_locale._fixup_ulcase', } def buildloaders(cls): - for constant, value in interp_locale.constants.iteritems(): + for constant, value in rlocale.constants.iteritems(): Module.interpleveldefs[constant] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py Fri Apr 23 02:41:35 2010 @@ -1,188 +1,72 @@ from pypy.rpython.tool import rffi_platform as platform -from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import rposix from pypy.rlib.rarithmetic import intmask from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib import rlocale +from pypy.module.exceptions.interp_exceptions import _new_exception, W_Exception +from pypy.rpython.lltypesystem import lltype, rffi -import sys - -HAVE_LANGINFO = sys.platform != 'win32' -HAVE_LIBINTL = sys.platform != 'win32' - -class CConfig: - includes = ['locale.h', 'limits.h'] - if HAVE_LANGINFO: - includes += ['langinfo.h'] - if HAVE_LIBINTL: - includes += ['libintl.h'] - if sys.platform == 'win32': - includes += ['windows.h'] - _compilation_info_ = ExternalCompilationInfo( - includes=includes, - ) - HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') - lconv = platform.Struct("struct lconv", [ - # Numeric (non-monetary) information. - ("decimal_point", rffi.CCHARP), # Decimal point character. - ("thousands_sep", rffi.CCHARP), # Thousands separator. - - ## Each element is the number of digits in each group; - ## elements with higher indices are farther left. - ## An element with value CHAR_MAX means that no further grouping is done. - ## An element with value 0 means that the previous element is used - ## for all groups farther left. */ - ("grouping", rffi.CCHARP), - - ## Monetary information. - - ## First three chars are a currency symbol from ISO 4217. - ## Fourth char is the separator. Fifth char is '\0'. - ("int_curr_symbol", rffi.CCHARP), - ("currency_symbol", rffi.CCHARP), # Local currency symbol. - ("mon_decimal_point", rffi.CCHARP), # Decimal point character. - ("mon_thousands_sep", rffi.CCHARP), # Thousands separator. - ("mon_grouping", rffi.CCHARP), # Like `grouping' element (above). - ("positive_sign", rffi.CCHARP), # Sign for positive values. - ("negative_sign", rffi.CCHARP), # Sign for negative values. - ("int_frac_digits", rffi.UCHAR), # Int'l fractional digits. - - ("frac_digits", rffi.UCHAR), # Local fractional digits. - ## 1 if currency_symbol precedes a positive value, 0 if succeeds. - ("p_cs_precedes", rffi.UCHAR), - ## 1 iff a space separates currency_symbol from a positive value. - ("p_sep_by_space", rffi.UCHAR), - ## 1 if currency_symbol precedes a negative value, 0 if succeeds. - ("n_cs_precedes", rffi.UCHAR), - ## 1 iff a space separates currency_symbol from a negative value. - ("n_sep_by_space", rffi.UCHAR), - - ## Positive and negative sign positions: - ## 0 Parentheses surround the quantity and currency_symbol. - ## 1 The sign string precedes the quantity and currency_symbol. - ## 2 The sign string follows the quantity and currency_symbol. - ## 3 The sign string immediately precedes the currency_symbol. - ## 4 The sign string immediately follows the currency_symbol. - ("p_sign_posn", rffi.UCHAR), - ("n_sign_posn", rffi.UCHAR), - ]) - - -constants = {} -constant_names = ( - 'LC_CTYPE', - 'LC_NUMERIC', - 'LC_TIME', - 'LC_COLLATE', - 'LC_MONETARY', - 'LC_MESSAGES', - 'LC_ALL', - 'LC_PAPER', - 'LC_NAME', - 'LC_ADDRESS', - 'LC_TELEPHONE', - 'LC_MEASUREMENT', - 'LC_IDENTIFICATION', - 'LC_MIN', - 'LC_MAX', - # from limits.h - 'CHAR_MAX', - ) - -for name in constant_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -langinfo_names = [] -if HAVE_LANGINFO: - # some of these consts have an additional #ifdef directives - # should we support them? - langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' - 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' - '_DATE_FMT'.split()) - for i in range(1, 8): - langinfo_names.append("DAY_%d" % i) - langinfo_names.append("ABDAY_%d" % i) - for i in range(1, 13): - langinfo_names.append("MON_%d" % i) - langinfo_names.append("ABMON_%d" % i) - -if sys.platform == 'win32': - langinfo_names.extend('LOCALE_USER_DEFAULT LOCALE_SISO639LANGNAME ' - 'LOCALE_SISO3166CTRYNAME LOCALE_IDEFAULTLANGUAGE ' - ''.split()) - - -for name in langinfo_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -class cConfig(object): - pass - -for k, v in platform.configure(CConfig).items(): - setattr(cConfig, k, v) +W_Error = _new_exception('Error', W_Exception, 'locale error') -# needed to export the constants inside and outside. see __init__.py -for name in constant_names: - value = getattr(cConfig, name) - if value is not None: - constants[name] = value - -for name in langinfo_names: - value = getattr(cConfig, name) - if value is not None and sys.platform != 'win32': - constants[name] = value - -locals().update(constants) - -HAVE_BIND_TEXTDOMAIN_CODESET = cConfig.HAVE_BIND_TEXTDOMAIN_CODESET - -def external(name, args, result, calling_conv='c'): - return rffi.llexternal(name, args, result, - compilation_info=CConfig._compilation_info_, - calling_conv=calling_conv) +import sys def make_error(space, msg): - w_module = space.getbuiltinmodule('_locale') - w_exception_class = space.getattr(w_module, space.wrap('Error')) - w_exception = space.call_function(w_exception_class, space.wrap(msg)) - return OperationError(w_exception_class, w_exception) + return OperationError(space.gettypeobject(W_Error.typedef), space.wrap(msg)) -_setlocale = external('setlocale', [rffi.INT, rffi.CCHARP], rffi.CCHARP) +def rewrap_error(space, e): + return OperationError(space.gettypeobject(W_Error.typedef), + space.wrap(e.message)) + +def _fixup_ulcase(space): + stringmod = space.call_function( + space.getattr(space.getbuiltinmodule('__builtin__'), + space.wrap('__import__')), space.wrap('string')) + # create uppercase map string + ul = [] + for c in xrange(256): + if rlocale.isupper(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('uppercase'), space.wrap(''.join(ul))) + + # create lowercase string + ul = [] + for c in xrange(256): + if rlocale.islower(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('lowercase'), space.wrap(''.join(ul))) + + # create letters string + ul = [] + for c in xrange(256): + if rlocale.isalpha(c): + ul.append(chr(c)) + space.setattr(stringmod, space.wrap('letters'), space.wrap(''.join(ul))) def setlocale(space, category, w_locale=None): "(integer,string=None) -> string. Activates/queries locale processing." - if cConfig.LC_MAX is not None: - if not cConfig.LC_MIN <= category <= cConfig.LC_MAX: - raise make_error(space, "invalid locale category") - if space.is_w(w_locale, space.w_None) or w_locale is None: - result = _setlocale(rffi.cast(rffi.INT, category), None) - if not result: - raise make_error(space, "locale query failed") + locale = None else: - locale = rffi.str2charp(space.str_w(w_locale)) - - result = _setlocale(rffi.cast(rffi.INT, category), locale) - if not result: - raise make_error(space, "unsupported locale setting") - - # record changes to LC_CTYPE - if category in (LC_CTYPE, LC_ALL): - w_module = space.getbuiltinmodule('_locale') - w_fun = space.getattr(w_module, space.wrap('_fixup_ulcase')) - space.call_function(w_fun) + locale = space.str_w(w_locale) + try: + result = rlocale.setlocale(category, locale) + except rlocale.LocaleError, e: + raise rewrap_error(space, e) + + # record changes to LC_CTYPE + if category in (rlocale.LC_CTYPE, rlocale.LC_ALL): + _fixup_ulcase(space) - return space.wrap(rffi.charp2str(result)) + return space.wrap(result) setlocale.unwrap_spec = [ObjSpace, int, W_Root] -_lconv = lltype.Ptr(cConfig.lconv) -_localeconv = external('localeconv', [], _lconv) +_lconv = lltype.Ptr(rlocale.cConfig.lconv) +_localeconv = rlocale.external('localeconv', [], _lconv) def _w_copy_grouping(space, text): groups = [ space.wrap(ord(group)) for group in text ] @@ -238,8 +122,8 @@ localeconv.unwrap_spec = [ObjSpace] -_strcoll = external('strcoll', [rffi.CCHARP, rffi.CCHARP], rffi.INT) -_wcscoll = external('wcscoll', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) +_strcoll = rlocale.external('strcoll', [rffi.CCHARP, rffi.CCHARP], rffi.INT) +_wcscoll = rlocale.external('wcscoll', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT) def strcoll(space, w_s1, w_s2): "string,string -> int. Compares two strings according to the locale." @@ -264,7 +148,7 @@ strcoll.unwrap_spec = [ObjSpace, W_Root, W_Root] -_strxfrm = external('strxfrm', +_strxfrm = rlocale.external('strxfrm', [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T) def strxfrm(space, s): @@ -287,27 +171,25 @@ strxfrm.unwrap_spec = [ObjSpace, str] -if HAVE_LANGINFO: - nl_item = rffi.INT - _nl_langinfo = external('nl_langinfo', [nl_item], rffi.CCHARP) +if rlocale.HAVE_LANGINFO: def nl_langinfo(space, key): """nl_langinfo(key) -> string Return the value for the locale information associated with key.""" - if key in constants.values(): - result = _nl_langinfo(rffi.cast(nl_item, key)) - return space.wrap(rffi.charp2str(result)) - raise OperationError(space.w_ValueError, - space.wrap("unsupported langinfo constant")) + try: + return space.wrap(rlocale.nl_langinfo(key)) + except ValueError: + raise OperationError(space.w_ValueError, + space.wrap("unsupported langinfo constant")) nl_langinfo.unwrap_spec = [ObjSpace, int] #___________________________________________________________________ # HAVE_LIBINTL dependence -if HAVE_LIBINTL: - _gettext = external('gettext', [rffi.CCHARP], rffi.CCHARP) +if rlocale.HAVE_LIBINTL: + _gettext = rlocale.external('gettext', [rffi.CCHARP], rffi.CCHARP) def gettext(space, msg): """gettext(msg) -> string @@ -316,7 +198,7 @@ gettext.unwrap_spec = [ObjSpace, str] - _dgettext = external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) + _dgettext = rlocale.external('dgettext', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def dgettext(space, w_domain, msg): """dgettext(domain, msg) -> string @@ -332,7 +214,7 @@ dgettext.unwrap_spec = [ObjSpace, W_Root, str] - _dcgettext = external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT], + _dcgettext = rlocale.external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT], rffi.CCHARP) def dcgettext(space, w_domain, msg, category): @@ -353,7 +235,7 @@ dcgettext.unwrap_spec = [ObjSpace, W_Root, str, int] - _textdomain = external('textdomain', [rffi.CCHARP], rffi.CCHARP) + _textdomain = rlocale.external('textdomain', [rffi.CCHARP], rffi.CCHARP) def textdomain(space, w_domain): """textdomain(domain) -> string @@ -370,7 +252,7 @@ textdomain.unwrap_spec = [ObjSpace, W_Root] - _bindtextdomain = external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP], + _bindtextdomain = rlocale.external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def bindtextdomain(space, domain, w_dir): @@ -392,10 +274,10 @@ bindtextdomain.unwrap_spec = [ObjSpace, str, W_Root] - _bind_textdomain_codeset = external('bind_textdomain_codeset', + _bind_textdomain_codeset = rlocale.external('bind_textdomain_codeset', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) - if HAVE_BIND_TEXTDOMAIN_CODESET: + if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET: def bind_textdomain_codeset(space, domain, w_codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" @@ -422,10 +304,10 @@ if sys.platform == 'win32': from pypy.rlib import rwin32 LCID = LCTYPE = rwin32.DWORD - GetACP = external('GetACP', + GetACP = rlocale.external('GetACP', [], rffi.INT, calling_conv='win') - GetLocaleInfo = external('GetLocaleInfoA', + GetLocaleInfo = rlocale.external('GetLocaleInfoA', [LCID, LCTYPE, rwin32.LPSTR, rffi.INT], rffi.INT, calling_conv='win') Modified: pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py (original) +++ pypy/branch/cpython-extension/pypy/module/_locale/test/test_locale.py Fri Apr 23 02:41:35 2010 @@ -7,9 +7,9 @@ def setup_class(cls): cls.space = space = gettestobjspace(usemodules=['_locale']) if sys.platform != 'win32': - cls.w_language_en = cls.space.wrap("en_US") - cls.w_language_utf8 = cls.space.wrap("en_US.UTF-8") - cls.w_language_pl = cls.space.wrap("pl_PL.UTF-8") + cls.w_language_en = cls.space.wrap("C") + cls.w_language_utf8 = cls.space.wrap("en_US.utf8") + cls.w_language_pl = cls.space.wrap("pl_PL.utf8") cls.w_encoding_pl = cls.space.wrap("utf-8") else: cls.w_language_en = cls.space.wrap("English_US") @@ -118,11 +118,11 @@ assert string.lowercase == lcase assert string.uppercase == ucase - if self.language_en != self.language_utf8: - _locale.setlocale(_locale.LC_ALL, self.language_en) + _locale.setlocale(_locale.LC_ALL, self.language_en) - assert string.lowercase != lcase - assert string.uppercase != ucase + # the asserts below are just plain wrong + # assert string.lowercase != lcase + # assert string.uppercase != ucase def test_localeconv(self): import _locale Modified: pypy/branch/cpython-extension/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/cpython-extension/pypy/module/_stackless/interp_coroutine.py Fri Apr 23 02:41:35 2010 @@ -15,7 +15,7 @@ experience to decide where to set the limits. """ -from pypy.interpreter.baseobjspace import Wrappable, UnpackValueError +from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w @@ -173,11 +173,8 @@ return nt([w_new_inst, nt(tup_base), nt(tup_state)]) def descr__setstate__(self, space, w_args): - try: - w_flags, w_state, w_thunk, w_parent = space.unpackiterable(w_args, - expected_length=4) - except UnpackValueError, e: - raise OperationError(space.w_ValueError, space.wrap(e.msg)) + w_flags, w_state, w_thunk, w_parent = space.unpackiterable(w_args, + expected_length=4) self.flags = space.int_w(w_flags) if space.is_w(w_parent, space.w_None): w_parent = self.w_getmain(space) @@ -188,11 +185,8 @@ if space.is_w(w_thunk, space.w_None): self.thunk = None else: - try: - w_func, w_args, w_kwds = space.unpackiterable(w_thunk, - expected_length=3) - except UnpackValueError, e: - raise OperationError(space.w_ValueError, space.wrap(e.msg)) + w_func, w_args, w_kwds = space.unpackiterable(w_thunk, + expected_length=3) args = Arguments.frompacked(space, w_args, w_kwds) self.bind(_AppThunk(space, self.costate, w_func, args)) Modified: pypy/branch/cpython-extension/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/bz2/interp_bz2.py (original) +++ pypy/branch/cpython-extension/pypy/module/bz2/interp_bz2.py Fri Apr 23 02:41:35 2010 @@ -222,7 +222,11 @@ head = "closed" else: head = "open" - info = "%s bz2.BZ2File '%s', mode '%s'" % (head, self.name, self.mode) + w_name = self.w_name + if w_name is None: + w_name = self.space.wrap('?') + info = "%s bz2.BZ2File %s, mode '%s'" % (head, self.getdisplayname(), + self.mode) return self.getrepr(self.space, info) file_bz2__repr__.unwrap_spec = ['self'] Modified: pypy/branch/cpython-extension/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/cpython-extension/pypy/module/posix/interp_posix.py Fri Apr 23 02:41:35 2010 @@ -20,7 +20,7 @@ except OSError, e: raise wrap_oserror(space, e, fname) return space.wrap(fd) -open.unwrap_spec = [ObjSpace, str, "c_int", "c_int"] +open.unwrap_spec = [ObjSpace, 'path', "c_int", "c_int"] def lseek(space, fd, pos, how): """Set the current position of a file descriptor. Return the new position. @@ -180,7 +180,7 @@ raise wrap_oserror(space, e, path) else: return build_stat_result(space, st) -stat.unwrap_spec = [ObjSpace, str] +stat.unwrap_spec = [ObjSpace, 'path'] def lstat(space, path): "Like stat(path), but do no follow symbolic links." @@ -190,7 +190,7 @@ raise wrap_oserror(space, e, path) else: return build_stat_result(space, st) -lstat.unwrap_spec = [ObjSpace, str] +lstat.unwrap_spec = [ObjSpace, 'path'] class StatState(object): def __init__(self, space): @@ -284,7 +284,7 @@ os.unlink(path) except OSError, e: raise wrap_oserror(space, e, path) -unlink.unwrap_spec = [ObjSpace, str] +unlink.unwrap_spec = [ObjSpace, 'path'] def remove(space, path): """Remove a file (same as unlink(path)).""" @@ -292,7 +292,7 @@ os.unlink(path) except OSError, e: raise wrap_oserror(space, e, path) -remove.unwrap_spec = [ObjSpace, str] +remove.unwrap_spec = [ObjSpace, 'path'] def _getfullpathname(space, path): """helper for ntpath.abspath """ Modified: pypy/branch/cpython-extension/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/cpython-extension/pypy/module/posix/test/test_posix2.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,6 @@ + +# -*- coding: utf-8 -*- + from pypy.objspace.std import StdObjSpace from pypy.tool.udir import udir from pypy.conftest import gettestobjspace @@ -35,10 +38,12 @@ if os.name == 'nt': py.test.skip("no sparse files on Windows") +GET_POSIX = "(): import %s as m ; return m" % os.name + class AppTestPosix: def setup_class(cls): cls.space = space - cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) + cls.w_posix = space.appexec([], GET_POSIX) cls.w_path = space.wrap(str(path)) cls.w_path2 = space.wrap(str(path2)) cls.w_pdir = space.wrap(str(pdir)) @@ -637,6 +642,46 @@ assert isinstance(f, file) assert f.read() == 'xxx' +class AppTestPosixUnicode: + + def setup_class(cls): + cls.space = space + cls.w_posix = space.appexec([], GET_POSIX) + if py.test.config.option.runappdirect: + # Can't change encoding + try: + u"?".encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + py.test.skip("encoding not good enough") + else: + cls.save_fs_encoding = space.sys.filesystemencoding + space.sys.filesystemencoding = "utf-8" + + def teardown_class(cls): + cls.space.sys.filesystemencoding = cls.save_fs_encoding + + def test_stat_unicode(self): + # test that passing unicode would not raise UnicodeDecodeError + try: + self.posix.stat(u"?") + except OSError: + pass + + def test_open_unicode(self): + # Ensure passing unicode doesn't raise UnicodeEncodeError + try: + self.posix.open(u"?", self.posix.O_WRONLY) + except OSError: + pass + + def test_remove_unicode(self): + # See 2 above ;) + try: + self.posix.remove(u"?") + except OSError: + pass + + class TestPexpect(object): # XXX replace with AppExpectTest class as soon as possible def setup_class(cls): Modified: pypy/branch/cpython-extension/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/cpython-extension/pypy/module/pypyjit/policy.py Fri Apr 23 02:41:35 2010 @@ -24,8 +24,7 @@ # gc_id operation if func.__name__ == 'id__ANY': return False - if mod == 'pypy.rlib.rbigint': - #if func.__name__ == '_bigint_true_divide': + if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff return False Modified: pypy/branch/cpython-extension/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/cpython-extension/pypy/module/pypyjit/test/test_policy.py Fri Apr 23 02:41:35 2010 @@ -10,6 +10,10 @@ from pypy.rlib.rbigint import rbigint assert not pypypolicy.look_inside_function(rbigint.lt.im_func) +def test_rlocale(): + from pypy.rlib.rlocale import setlocale + assert not pypypolicy.look_inside_function(setlocale) + def test_geninterp(): d = {'_geninterp_': True} exec """def f(): Modified: pypy/branch/cpython-extension/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/cpython-extension/pypy/module/signal/interp_signal.py Fri Apr 23 02:41:35 2010 @@ -39,7 +39,10 @@ # pointless and a performance issue # don't use rffi.LONGP because the JIT doesn't support raw arrays so far -LONG_STRUCT = lltype.Struct('LONG_STRUCT', ('value', lltype.Signed)) +struct_name = 'pypysig_long_struct' +LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), + hints={'c_name' : struct_name, 'external' : 'C'}) +del struct_name pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], lltype.Ptr(LONG_STRUCT), _nowrapper=True, @@ -51,10 +54,10 @@ class SignalActionFlag(AbstractActionFlag): def get(self): p = pypysig_getaddr_occurred() - return p.value + return p.c_value def set(self, value): p = pypysig_getaddr_occurred() - p.value = value + p.c_value = value class CheckSignalAction(AsyncAction): Modified: pypy/branch/cpython-extension/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/sys/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/sys/__init__.py Fri Apr 23 02:41:35 2010 @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.error import OperationError class Module(MixedModule): @@ -10,6 +10,7 @@ self.recursionlimit = 100 self.w_default_encoder = None self.defaultencoding = "ascii" + self.filesystemencoding = None interpleveldefs = { '__name__' : '(space.wrap("sys"))', @@ -68,13 +69,14 @@ 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', -} + 'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding', + } + appleveldefs = { 'excepthook' : 'app.excepthook', '__excepthook__' : 'app.excepthook', 'exit' : 'app.exit', 'exitfunc' : 'app.exitfunc', - 'getfilesystemencoding' : 'app.getfilesystemencoding', 'callstats' : 'app.callstats', 'copyright' : 'app.copyright_str', } @@ -84,6 +86,10 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) + def startup(self, space): + from pypy.module.sys.interp_encoding import _getfilesystemencoding + self.filesystemencoding = _getfilesystemencoding(space) + def getmodule(self, name): space = self.space w_modules = self.get('modules') Modified: pypy/branch/cpython-extension/pypy/module/sys/app.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/sys/app.py (original) +++ pypy/branch/cpython-extension/pypy/module/sys/app.py Fri Apr 23 02:41:35 2010 @@ -26,24 +26,12 @@ #import __builtin__ -def getfilesystemencoding(): - """Return the encoding used to convert Unicode filenames in -operating system filenames. - """ - if sys.platform == "win32": - encoding = "mbcs" - elif sys.platform == "darwin": - encoding = "utf-8" - else: - encoding = None - return encoding - def callstats(): """Not implemented.""" return None copyright_str = """ -Copyright 2003-2009 PyPy development team. +Copyright 2003-2010 PyPy development team. All rights reserved. For further information see http://www.codespeak.net/pypy. Modified: pypy/branch/cpython-extension/pypy/module/sys/interp_encoding.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/sys/interp_encoding.py (original) +++ pypy/branch/cpython-extension/pypy/module/sys/interp_encoding.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,6 @@ +import sys +from pypy.rlib import rlocale + def getdefaultencoding(space): """Return the current default string encoding used by the Unicode implementation.""" @@ -22,3 +25,31 @@ w_encoder = space.getitem(w_functuple, space.wrap(0)) space.sys.w_default_encoder = w_encoder # cache it return w_encoder + +if sys.platform == "win32": + base_encoding = "mbcs" +elif sys.platform == "darwin": + base_encoding = "utf-8" +else: + base_encoding = None + +def _getfilesystemencoding(space): + encoding = base_encoding + if rlocale.HAVE_LANGINFO and rlocale.CODESET: + oldlocale = rlocale.setlocale(rlocale.LC_CTYPE, None) + rlocale.setlocale(rlocale.LC_CTYPE, "") + loc_codeset = rlocale.nl_langinfo(rlocale.CODESET) + if loc_codeset: + codecmod = space.getbuiltinmodule('_codecs') + w_res = space.call_function(space.getattr(codecmod, + space.wrap('lookup')), + space.wrap(loc_codeset)) + if space.is_true(w_res): + encoding = loc_codeset + return encoding + +def getfilesystemencoding(space): + """Return the encoding used to convert Unicode filenames in + operating system filenames. + """ + return space.wrap(space.sys.filesystemencoding) Modified: pypy/branch/cpython-extension/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/sys/test/test_sysmodule.py Fri Apr 23 02:41:35 2010 @@ -3,6 +3,7 @@ from pypy.conftest import option from py.test import raises from pypy.interpreter.gateway import app2interp_temp +import sys def init_globals_via_builtins_hack(space): space.appexec([], """(): @@ -24,6 +25,7 @@ def setup_class(cls): cls.w_appdirect = cls.space.wrap(option.runappdirect) + cls.w_filesystemenc = cls.space.wrap(sys.getfilesystemencoding()) def test_sys_in_modules(self): import sys @@ -108,6 +110,10 @@ assert isinstance(sys.stderr, file) assert isinstance(sys.stdin, file) + def test_getfilesystemencoding(self): + import sys + assert sys.getfilesystemencoding() == self.filesystemenc + class AppTestSysModulePortedFromCPython: def setup_class(cls): Modified: pypy/branch/cpython-extension/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/cpython-extension/pypy/module/termios/interp_termios.py Fri Apr 23 02:41:35 2010 @@ -28,14 +28,8 @@ return OperationError(w_exception_class, w_exception) def tcsetattr(space, fd, when, w_attributes): - from pypy.interpreter.baseobjspace import UnpackValueError - try: - w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ - space.unpackiterable(w_attributes, expected_length=7) - except UnpackValueError, e: - raise OperationError( - space.w_TypeError, - space.wrap("tcsetattr, arg 3: must be 7 element list")) + w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed, w_cc = \ + space.unpackiterable(w_attributes, expected_length=7) w_builtin = space.getbuiltinmodule('__builtin__') cc = [] for w_c in space.unpackiterable(w_cc): Modified: pypy/branch/cpython-extension/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/flowcontext.py Fri Apr 23 02:41:35 2010 @@ -1,18 +1,14 @@ import collections +import sys from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError -from pypy.interpreter import pyframe +from pypy.interpreter import pyframe, nestedscope from pypy.interpreter.argument import ArgumentsForTranslation +from pypy.objspace.flow import operation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState from pypy.rlib import jit - - -class OperationThatShouldNotBePropagatedError(OperationError): - pass - -class ImplicitOperationError(OperationError): - pass +from pypy.tool.stdlib_opcode import host_bytecode_spec class StopFlowing(Exception): pass @@ -200,8 +196,8 @@ if closure is None: self.closure = None else: - from pypy.interpreter.nestedscope import Cell - self.closure = [Cell(Constant(value)) for value in closure] + self.closure = [nestedscope.Cell(Constant(value)) + for value in closure] frame = self.create_frame() formalargcount = code.getformalargcount() arg_list = [Variable() for i in range(formalargcount)] @@ -270,13 +266,13 @@ self.crnt_frame = None self.topframeref = old_frameref - except OperationThatShouldNotBePropagatedError, e: + except operation.OperationThatShouldNotBePropagatedError, e: raise Exception( 'found an operation that always raises %s: %s' % ( self.space.unwrap(e.w_type).__name__, self.space.unwrap(e.get_w_value(self.space)))) - except ImplicitOperationError, e: + except operation.ImplicitOperationError, e: if isinstance(e.w_type, Constant): exc_cls = e.w_type.value else: @@ -379,7 +375,7 @@ def sys_exc_info(self): operr = ExecutionContext.sys_exc_info(self) - if isinstance(operr, ImplicitOperationError): + if isinstance(operr, operation.ImplicitOperationError): # re-raising an implicit operation makes it an explicit one w_value = operr.get_w_value(self.space) operr = OperationError(operr.w_type, w_value) @@ -399,6 +395,90 @@ stack_items_w[i] = w_new class FlowSpaceFrame(pyframe.PyFrame): + """ + Execution of host (CPython) opcodes. + """ + bytecode_spec = host_bytecode_spec + opcode_method_names = host_bytecode_spec.method_names + opcodedesc = host_bytecode_spec.opcodedesc + opdescmap = host_bytecode_spec.opdescmap + HAVE_ARGUMENT = host_bytecode_spec.HAVE_ARGUMENT + + def BUILD_MAP(self, itemcount, next_instr): + if sys.version_info >= (2, 6): + # We could pre-allocate a dict here + # but for the moment this code is not translated. + pass + else: + if itemcount != 0: + raise BytecodeCorruption + w_dict = self.space.newdict() + self.pushvalue(w_dict) + + def STORE_MAP(self, zero, next_instr): + if sys.version_info >= (2, 6): + w_key = self.popvalue() + w_value = self.popvalue() + w_dict = self.peekvalue() + self.space.setitem(w_dict, w_key, w_value) + else: + raise BytecodeCorruption + + def POP_JUMP_IF_FALSE(self, jumpto, next_instr): + w_cond = self.popvalue() + if not self.space.is_true(w_cond): + next_instr = jumpto + return next_instr + + def POP_JUMP_IF_TRUE(self, jumpto, next_instr): + w_cond = self.popvalue() + if self.space.is_true(w_cond): + return jumpto + return next_instr + + def JUMP_IF_FALSE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if not self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def JUMP_IF_TRUE_OR_POP(self, jumpto, next_instr): + w_cond = self.peekvalue() + if self.space.is_true(w_cond): + return jumpto + self.popvalue() + return next_instr + + def LIST_APPEND(self, oparg, next_instr): + w = self.popvalue() + if sys.version_info < (2, 7): + v = self.popvalue() + else: + v = self.peekvalue(oparg - 1) + self.space.call_method(v, 'append', w) + + # XXX Unimplemented 2.7 opcodes ---------------- + + # Set literals, set comprehensions + + def BUILD_SET(self, oparg, next_instr): + raise NotImplementedError("BUILD_SET") + + def SET_ADD(self, oparg, next_instr): + raise NotImplementedError("SET_ADD") + + # Dict comprehensions + + def MAP_ADD(self, oparg, next_instr): + raise NotImplementedError("MAP_ADD") + + # `with` statement + + def SETUP_WITH(self, oparg, next_instr): + raise NotImplementedError("SETUP_WITH") + + def make_arguments(self, nargs): return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) def argument_factory(self, *args): @@ -406,7 +486,7 @@ def handle_operation_error(self, ec, operr, *args, **kwds): # see test_propagate_attribute_error for why this is here - if isinstance(operr, OperationThatShouldNotBePropagatedError): + if isinstance(operr, operation.OperationThatShouldNotBePropagatedError): raise operr return pyframe.PyFrame.handle_operation_error(self, ec, operr, *args, **kwds) Modified: pypy/branch/cpython-extension/pypy/objspace/flow/model.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/model.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/model.py Fri Apr 23 02:41:35 2010 @@ -322,6 +322,14 @@ self.concretetype = concretetype +class UnwrapException(Exception): + """Attempted to unwrap a Variable.""" + +class WrapException(Exception): + """Attempted wrapping of a type that cannot sanely appear in flow graph or + during its construction""" + + class SpaceOperation(object): __slots__ = "opname args result offset".split() Modified: pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/objspace.py Fri Apr 23 02:41:35 2010 @@ -1,22 +1,19 @@ # ______________________________________________________________________ -import sys, operator, types +import __builtin__ +import sys +import operator +import types +from pypy.tool import error from pypy.interpreter.baseobjspace import ObjSpace, Wrappable from pypy.interpreter.pycode import PyCode, cpython_code_signature from pypy.interpreter.module import Module from pypy.interpreter.error import OperationError +from pypy.interpreter import pyframe, argument from pypy.objspace.flow.model import * -from pypy.objspace.flow import flowcontext -from pypy.objspace.flow.operation import FunctionByName +from pypy.objspace.flow import flowcontext, operation, specialcase from pypy.rlib.unroll import unrolling_iterable, _unroller -from pypy.rlib import rstackovf +from pypy.rlib import rstackovf, rarithmetic -debug = 0 - -class UnwrapException(Exception): - "Attempted to unwrap a Variable." - -class WrapException(Exception): - """Attempted wrapping of a type that cannot sanely appear in flow graph or during its construction""" # method-wrappers have not enough introspection in CPython if hasattr(complex.real.__get__, 'im_self'): @@ -24,6 +21,21 @@ else: type_with_bad_introspection = type(complex.real.__get__) +# the following gives us easy access to declare more for applications: +NOT_REALLY_CONST = { + Constant(sys): { + Constant('maxint'): True, + Constant('maxunicode'): True, + Constant('api_version'): True, + Constant('exit'): True, + Constant('exc_info'): True, + Constant('getrefcount'): True, + Constant('getdefaultencoding'): True, + # this is an incomplete list of true constants. + # if we add much more, a dedicated class + # might be considered for special objects. + } + } # ______________________________________________________________________ class FlowObjSpace(ObjSpace): @@ -32,15 +44,16 @@ the space operations that the interpreter generates when it interprets (the bytecode of) some function. """ - + full_exceptions = False do_imports_immediately = True + FrameClass = flowcontext.FlowSpaceFrame def initialize(self): - import __builtin__ self.concrete_mode = 1 self.w_None = Constant(None) - self.builtin = Module(self, Constant('__builtin__'), Constant(__builtin__.__dict__)) + self.builtin = Module(self, Constant('__builtin__'), + Constant(__builtin__.__dict__)) def pick_builtin(w_globals): return self.builtin self.builtin.pick_builtin = pick_builtin @@ -125,9 +138,8 @@ def uint_w(self, w_obj): if isinstance(w_obj, Constant): - from pypy.rlib.rarithmetic import r_uint val = w_obj.value - if type(val) is not r_uint: + if type(val) is not rarithmetic.r_uint: raise TypeError("expected unsigned: " + repr(w_obj)) return val return self.unwrap(w_obj) @@ -191,7 +203,6 @@ def setup_executioncontext(self, ec): self.executioncontext = ec - from pypy.objspace.flow import specialcase specialcase.setup(self) def exception_match(self, w_exc_type, w_check_class): @@ -254,15 +265,15 @@ graph.defaults = func.func_defaults or () self.setup_executioncontext(ec) - from pypy.tool.error import FlowingError, format_global_error - try: ec.build_flow() - except FlowingError, a: + except error.FlowingError, a: # attach additional source info to AnnotatorError _, _, tb = sys.exc_info() - e = FlowingError(format_global_error(ec.graph, ec.crnt_offset, str(a))) - raise FlowingError, e, tb + formated = error.format_global_error(ec.graph, ec.crnt_offset, + str(a)) + e = error.FlowingError(formated) + raise error.FlowingError, e, tb checkgraph(graph) return graph @@ -300,7 +311,7 @@ def do_operation_with_implicit_exceptions(self, name, *args_w): w_result = self.do_operation(name, *args_w) - self.handle_implicit_exceptions(implicit_exceptions.get(name)) + self.handle_implicit_exceptions(operation.implicit_exceptions.get(name)) return w_result def is_true(self, w_obj): @@ -346,8 +357,8 @@ if outcome is StopIteration: raise OperationError(self.w_StopIteration, w_exc_value) elif outcome is RuntimeError: - raise flowcontext.ImplicitOperationError(Constant(RuntimeError), - w_exc_value) + raise operation.ImplicitOperationError(Constant(RuntimeError), + w_exc_value) else: return w_item @@ -365,9 +376,8 @@ w_key, w_val) def call_function(self, w_func, *args_w): - from pypy.interpreter.argument import ArgumentsForTranslation nargs = len(args_w) - args = ArgumentsForTranslation(self, list(args_w)) + args = argument.ArgumentsForTranslation(self, list(args_w)) return self.call_args(w_func, args) def call_args(self, w_callable, args): @@ -403,7 +413,7 @@ # raise SomeError(x) # # as shown by test_objspace.test_raise3. - + exceptions = [Exception] # *any* exception by default if isinstance(w_callable, Constant): c = w_callable.value @@ -413,7 +423,7 @@ types.ClassType, types.TypeType)) and c.__module__ in ['__builtin__', 'exceptions']): - exceptions = implicit_exceptions.get(c, None) + exceptions = operation.implicit_exceptions.get(c) self.handle_implicit_exceptions(exceptions) return w_res @@ -441,11 +451,9 @@ #if outcome is not Exception: #w_exc_cls = Constant(outcome) Now done by guessexception itself #pass - raise flowcontext.ImplicitOperationError(w_exc_cls, - w_exc_value) + raise operation.ImplicitOperationError(w_exc_cls, w_exc_value) def w_KeyboardInterrupt(self): - # XXX XXX Ha Ha # the reason to do this is: if you interrupt the flowing of a function # with the bytecode interpreter will raise an applevel # KeyboardInterrupt and you will get an AttributeError: space does not @@ -458,84 +466,8 @@ raise RuntimeError("the interpreter raises RuntimeError during " "flow graph construction") w_RuntimeError = prebuilt_recursion_error = property(w_RuntimeError) +operation.add_operations(FlowObjSpace) -# the following gives us easy access to declare more for applications: -NOT_REALLY_CONST = { - Constant(sys): { - Constant('maxint'): True, - Constant('maxunicode'): True, - Constant('api_version'): True, - Constant('exit'): True, - Constant('exc_info'): True, - Constant('getrefcount'): True, - Constant('getdefaultencoding'): True, - # this is an incomplete list of true constants. - # if we add much more, a dedicated class - # might be considered for special objects. - } - } - -# ______________________________________________________________________ - -op_appendices = { - OverflowError: 'ovf', - IndexError: 'idx', - KeyError: 'key', - AttributeError: 'att', - TypeError: 'typ', - ZeroDivisionError: 'zer', - ValueError: 'val', - } - -implicit_exceptions = { - int: [ValueError], # built-ins that can always raise exceptions - float: [ValueError], - chr: [ValueError], - unichr: [ValueError], - # specifying IndexError, and KeyError beyond Exception, - # allows the annotator to be more precise, see test_reraiseAnything/KeyError in - # the annotator tests - 'getitem': [IndexError, KeyError, Exception], - 'setitem': [IndexError, KeyError, Exception], - 'delitem': [IndexError, KeyError, Exception], - 'contains': [Exception], # from an r_dict - } - -def _add_exceptions(names, exc): - for name in names.split(): - lis = implicit_exceptions.setdefault(name, []) - if exc in lis: - raise ValueError, "your list is causing duplication!" - lis.append(exc) - assert exc in op_appendices - -def _add_except_ovf(names): - # duplicate exceptions and add OverflowError - for name in names.split(): - lis = implicit_exceptions.setdefault(name, [])[:] - lis.append(OverflowError) - implicit_exceptions[name+"_ovf"] = lis - -for _name in 'getattr', 'delattr': - _add_exceptions(_name, AttributeError) -for _name in 'iter', 'coerce': - _add_exceptions(_name, TypeError) -del _name - -_add_exceptions("""div mod divmod truediv floordiv pow - inplace_div inplace_mod inplace_divmod inplace_truediv - inplace_floordiv inplace_pow""", ZeroDivisionError) -_add_exceptions("""pow inplace_pow lshift inplace_lshift rshift - inplace_rshift""", ValueError) -_add_exceptions("""truediv divmod - inplace_add inplace_sub inplace_mul inplace_truediv - inplace_floordiv inplace_div inplace_mod inplace_pow - inplace_lshift""", OverflowError) # without a _ovf version -_add_except_ovf("""neg abs add sub mul - floordiv div mod pow lshift""") # with a _ovf version -_add_exceptions("""pow""", - OverflowError) # for the float case -del _add_exceptions, _add_except_ovf def extract_cell_content(c): """Get the value contained in a CPython 'cell', as read through @@ -558,127 +490,5 @@ return x.other # crashes if the cell is actually empty except AttributeError: raise ValueError("empty cell") - -def make_op(name, symbol, arity, specialnames): - if hasattr(FlowObjSpace, name): - return # Shouldn't do it - - import __builtin__ - - op = None - skip = False - arithmetic = False - - if name.startswith('del') or name.startswith('set') or name.startswith('inplace_'): - # skip potential mutators - if debug: print "Skip", name - skip = True - elif name in ['id', 'hash', 'iter', 'userdel']: - # skip potential runtime context dependecies - if debug: print "Skip", name - skip = True - elif name in ['repr', 'str']: - rep = getattr(__builtin__, name) - def op(obj): - s = rep(obj) - if s.find("at 0x") > -1: - print >>sys.stderr, "Warning: captured address may be awkward" - return s - else: - op = FunctionByName[name] - arithmetic = (name + '_ovf') in FunctionByName - - if not op: - if not skip: - if debug: print >> sys.stderr, "XXX missing operator:", name - else: - if debug: print "Can constant-fold operation: %s" % name - - def generic_operator(self, *args_w): - assert len(args_w) == arity, name+" got the wrong number of arguments" - if op: - args = [] - for w_arg in args_w: - try: - arg = self.unwrap_for_computation(w_arg) - except UnwrapException: - break - else: - args.append(arg) - else: - # All arguments are constants: call the operator now - #print >> sys.stderr, 'Constant operation', op - try: - result = op(*args) - except: - etype, evalue, etb = sys.exc_info() - msg = "generated by a constant operation: %s%r" % ( - name, tuple(args)) - raise flowcontext.OperationThatShouldNotBePropagatedError( - self.wrap(etype), self.wrap(msg)) - else: - # don't try to constant-fold operations giving a 'long' - # result. The result is probably meant to be sent to - # an intmask(), but the 'long' constant confuses the - # annotator a lot. - if arithmetic and type(result) is long: - pass - # don't constant-fold getslice on lists, either - elif name == 'getslice' and type(result) is list: - pass - # otherwise, fine - else: - try: - return self.wrap(result) - except WrapException: - # type cannot sanely appear in flow graph, - # store operation with variable result instead - pass - - #print >> sys.stderr, 'Variable operation', name, args_w - w_result = self.do_operation_with_implicit_exceptions(name, *args_w) - return w_result - - setattr(FlowObjSpace, name, generic_operator) - -for line in ObjSpace.MethodTable: - make_op(*line) - -""" -This is just a placeholder for some code I'm checking in elsewhere. -It is provenly possible to determine constantness of certain expressions -a little later. I introduced this a bit too early, together with tieing -this to something being global, which was a bad idea. -The concept is still valid, and it can be used to force something to -be evaluated immediately because it is supposed to be a constant. -One good possible use of this is loop unrolling. -This will be found in an 'experimental' folder with some use cases. -""" - -def override(): - def getattr(self, w_obj, w_name): - # handling special things like sys - # unfortunately this will never vanish with a unique import logic :-( - if w_obj in self.not_really_const: - const_w = self.not_really_const[w_obj] - if w_name not in const_w: - return self.do_operation_with_implicit_exceptions('getattr', w_obj, w_name) - return self.regular_getattr(w_obj, w_name) - - FlowObjSpace.regular_getattr = FlowObjSpace.getattr - FlowObjSpace.getattr = getattr - - # protect us from globals write access - def setitem(self, w_obj, w_key, w_val): - ec = self.getexecutioncontext() - if not (ec and w_obj is ec.w_globals): - return self.regular_setitem(w_obj, w_key, w_val) - raise SyntaxError, "attempt to modify global attribute %r in %r" % (w_key, ec.graph.func) - - FlowObjSpace.regular_setitem = FlowObjSpace.setitem - FlowObjSpace.setitem = setitem - -override() - # ______________________________________________________________________ # End of objspace.py Modified: pypy/branch/cpython-extension/pypy/objspace/flow/operation.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/operation.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/operation.py Fri Apr 23 02:41:35 2010 @@ -2,10 +2,25 @@ This module defines mappings between operation names and Python's built-in functions (or type constructors) implementing them. """ + +import __builtin__ +import __future__ +import operator +import types +import sys from pypy.interpreter.baseobjspace import ObjSpace -import operator, types, __future__ +from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import compile2 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift +from pypy.objspace.flow import model + + +class OperationThatShouldNotBePropagatedError(OperationError): + pass + +class ImplicitOperationError(OperationError): + pass + FunctionByName = {} # dict {"operation_name": } OperationName = {} # dict {: "operation_name"} @@ -216,8 +231,6 @@ ] def setup(): - if not hasattr(operator, 'is_'): # Python 2.2 - Table.append(('is_', lambda x, y: x is y)) # insert all operators for line in ObjSpace.MethodTable: name = line[0] @@ -235,4 +248,183 @@ Arity[name] = line[2] assert name in FunctionByName setup() -del Table # INTERNAL ONLY, use the dicts declared at the top of the file +del Table, setup # INTERNAL ONLY, use the dicts declared at the top of the file + +op_appendices = { + OverflowError: 'ovf', + IndexError: 'idx', + KeyError: 'key', + AttributeError: 'att', + TypeError: 'typ', + ZeroDivisionError: 'zer', + ValueError: 'val', + } + +implicit_exceptions = { + int: [ValueError], # built-ins that can always raise exceptions + float: [ValueError], + chr: [ValueError], + unichr: [ValueError], + unicode: [UnicodeDecodeError], + # specifying IndexError, and KeyError beyond Exception, + # allows the annotator to be more precise, see test_reraiseAnything/KeyError in + # the annotator tests + 'getitem': [IndexError, KeyError, Exception], + 'setitem': [IndexError, KeyError, Exception], + 'delitem': [IndexError, KeyError, Exception], + 'contains': [Exception], # from an r_dict + } + +def _add_exceptions(names, exc): + for name in names.split(): + lis = implicit_exceptions.setdefault(name, []) + if exc in lis: + raise ValueError, "your list is causing duplication!" + lis.append(exc) + assert exc in op_appendices + +def _add_except_ovf(names): + # duplicate exceptions and add OverflowError + for name in names.split(): + lis = implicit_exceptions.setdefault(name, [])[:] + lis.append(OverflowError) + implicit_exceptions[name+"_ovf"] = lis + +for _name in 'getattr', 'delattr': + _add_exceptions(_name, AttributeError) +for _name in 'iter', 'coerce': + _add_exceptions(_name, TypeError) +del _name + +_add_exceptions("""div mod divmod truediv floordiv pow + inplace_div inplace_mod inplace_divmod inplace_truediv + inplace_floordiv inplace_pow""", ZeroDivisionError) +_add_exceptions("""pow inplace_pow lshift inplace_lshift rshift + inplace_rshift""", ValueError) +_add_exceptions("""truediv divmod + inplace_add inplace_sub inplace_mul inplace_truediv + inplace_floordiv inplace_div inplace_mod inplace_pow + inplace_lshift""", OverflowError) # without a _ovf version +_add_except_ovf("""neg abs add sub mul + floordiv div mod pow lshift""") # with a _ovf version +_add_exceptions("""pow""", + OverflowError) # for the float case +del _add_exceptions, _add_except_ovf + +def make_op(fs, name, symbol, arity, specialnames): + if hasattr(fs, name): + return + + op = None + skip = False + arithmetic = False + + if (name.startswith('del') or + name.startswith('set') or + name.startswith('inplace_')): + # skip potential mutators + skip = True + elif name in ('id', 'hash', 'iter', 'userdel'): + # skip potential runtime context dependecies + skip = True + elif name in ('repr', 'str'): + rep = getattr(__builtin__, name) + def op(obj): + s = rep(obj) + if "at 0x" in s: + print >>sys.stderr, "Warning: captured address may be awkward" + return s + else: + op = FunctionByName[name] + arithmetic = (name + '_ovf') in FunctionByName + + if not op and not skip: + raise ValueError("XXX missing operator: %s" % (name,)) + + def generic_operator(self, *args_w): + assert len(args_w) == arity, name + " got the wrong number of arguments" + if op: + args = [] + for w_arg in args_w: + try: + arg = self.unwrap_for_computation(w_arg) + except model.UnwrapException: + break + else: + args.append(arg) + else: + # All arguments are constants: call the operator now + try: + result = op(*args) + except: + etype, evalue, etb = sys.exc_info() + msg = "generated by a constant operation: %s%r" % ( + name, tuple(args)) + raise OperationThatShouldNotBePropagatedError( + self.wrap(etype), self.wrap(msg)) + else: + # don't try to constant-fold operations giving a 'long' + # result. The result is probably meant to be sent to + # an intmask(), but the 'long' constant confuses the + # annotator a lot. + if arithmetic and type(result) is long: + pass + # don't constant-fold getslice on lists, either + elif name == 'getslice' and type(result) is list: + pass + # otherwise, fine + else: + try: + return self.wrap(result) + except model.WrapException: + # type cannot sanely appear in flow graph, + # store operation with variable result instead + pass + w_result = self.do_operation_with_implicit_exceptions(name, *args_w) + return w_result + + setattr(fs, name, generic_operator) + + +""" +This is just a placeholder for some code I'm checking in elsewhere. +It is provenly possible to determine constantness of certain expressions +a little later. I introduced this a bit too early, together with tieing +this to something being global, which was a bad idea. +The concept is still valid, and it can be used to force something to +be evaluated immediately because it is supposed to be a constant. +One good possible use of this is loop unrolling. +This will be found in an 'experimental' folder with some use cases. +""" + +def special_overrides(fs): + def getattr(self, w_obj, w_name): + # handling special things like sys + # unfortunately this will never vanish with a unique import logic :-( + if w_obj in self.not_really_const: + const_w = self.not_really_const[w_obj] + if w_name not in const_w: + return self.do_operation_with_implicit_exceptions('getattr', + w_obj, w_name) + return self.regular_getattr(w_obj, w_name) + + fs.regular_getattr = fs.getattr + fs.getattr = getattr + + # protect us from globals write access + def setitem(self, w_obj, w_key, w_val): + ec = self.getexecutioncontext() + if not (ec and w_obj is ec.w_globals): + return self.regular_setitem(w_obj, w_key, w_val) + raise SyntaxError("attempt to modify global attribute %r in %r" + % (w_key, ec.graph.func)) + + fs.regular_setitem = fs.setitem + fs.setitem = setitem + + +def add_operations(fs): + """Add function operations to the flow space.""" + for line in ObjSpace.MethodTable: + make_op(fs, *line) + special_overrides(fs) Modified: pypy/branch/cpython-extension/pypy/objspace/flow/specialcase.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/specialcase.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/specialcase.py Fri Apr 23 02:41:35 2010 @@ -1,9 +1,9 @@ -from pypy.objspace.flow.objspace import UnwrapException -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Constant, UnwrapException from pypy.objspace.flow.operation import OperationName, Arity from pypy.interpreter.gateway import ApplevelClass from pypy.interpreter.error import OperationError from pypy.tool.cache import Cache +import py def sc_import(space, fn, args): args_w, kwds_w = args.unpack() @@ -73,7 +73,7 @@ if app.filename is not None: dic['__file__'] = app.filename dic['__name__'] = app.modname - exec app.code in dic + exec py.code.Source(app.source).compile() in dic return dic _build = staticmethod(_build) Modified: pypy/branch/cpython-extension/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/flow/test/test_objspace.py Fri Apr 23 02:41:35 2010 @@ -1,12 +1,14 @@ import new import py from pypy.objspace.flow.model import Constant, Block, Link, Variable, traverse -from pypy.objspace.flow.model import flatten, mkentrymap +from pypy.objspace.flow.model import flatten, mkentrymap, c_last_exception from pypy.interpreter.argument import Arguments from pypy.translator.simplify import simplify_graph from pypy.objspace.flow.objspace import FlowObjSpace -from pypy.objspace.flow import objspace +from pypy.objspace.flow import objspace, flowcontext from pypy import conftest +from pypy.tool.stdlib_opcode import bytecode_spec +from pypy.interpreter.pyframe import PyFrame import os import operator @@ -728,7 +730,18 @@ return unicode("1234") graph = self.codetest(myfunc) assert graph.startblock.exits[0].target is graph.returnblock - + + def test_unicode(self): + def myfunc(n): + try: + return unicode(chr(n)) + except UnicodeDecodeError: + return None + graph = self.codetest(myfunc) + simplify_graph(graph) + assert graph.startblock.exitswitch == c_last_exception + assert graph.startblock.exits[0].target is graph.returnblock + assert graph.startblock.exits[1].target is graph.returnblock def test_getitem(self): def f(c, x): @@ -826,23 +839,34 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - class X: - def m(self): - return 3 - - def f(): - x = X() - return x.m() - - # this code is generated by pypy-c when compiling above f - pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' - new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) - f2 = new.function(new_c, locals(), 'f') - - graph = self.codetest(f2) - all_ops = self.all_operations(graph) - assert all_ops['simple_call'] == 2 - assert all_ops['getattr'] == 1 + flow_meth_names = flowcontext.FlowSpaceFrame.opcode_method_names + pyframe_meth_names = PyFrame.opcode_method_names + for name in ['CALL_METHOD', 'LOOKUP_METHOD']: + num = bytecode_spec.opmap[name] + locals()['old_' + name] = flow_meth_names[num] + flow_meth_names[num] = pyframe_meth_names[num] + try: + class X: + def m(self): + return 3 + + def f(): + x = X() + return x.m() + + # this code is generated by pypy-c when compiling above f + pypy_code = 't\x00\x00\x83\x00\x00}\x00\x00|\x00\x00\x91\x02\x00\x92\x00\x00Sd\x00\x00S' + new_c = self.monkey_patch_code(f.func_code, 3, 3, pypy_code, ('X', 'x', 'm'), ('x',)) + f2 = new.function(new_c, locals(), 'f') + + graph = self.codetest(f2) + all_ops = self.all_operations(graph) + assert all_ops['simple_call'] == 2 + assert all_ops['getattr'] == 1 + finally: + for name in ['CALL_METHOD', 'LOOKUP_METHOD']: + num = bytecode_spec.opmap[name] + flow_meth_names[num] = locals()['old_' + name] def test_generator(self): def f(): Modified: pypy/branch/cpython-extension/pypy/objspace/std/basestringtype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/basestringtype.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/basestringtype.py Fri Apr 23 02:41:35 2010 @@ -1,9 +1,7 @@ -from pypy.objspace.std.stdtypedef import * +from pypy.objspace.std.stdtypedef import StdTypeDef -# ____________________________________________________________ - basestring_typedef = StdTypeDef("basestring", - __doc__ = '''Type basestring cannot be instantiated; it is the base for str and unicode.''' + __doc__ = ("basestring cannot be instantiated; " + "it is the base for str and unicode.") ) - Modified: pypy/branch/cpython-extension/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/dictmultiobject.py Fri Apr 23 02:41:35 2010 @@ -102,9 +102,6 @@ else: return None - def set_str_keyed_item(w_dict, key, w_value, shadows_type=True): - w_dict.setitem_str(key, w_value, shadows_type) - # _________________________________________________________________ # implementation methods def impl_getitem(self, w_key): Modified: pypy/branch/cpython-extension/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/formatting.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/formatting.py Fri Apr 23 02:41:35 2010 @@ -329,7 +329,8 @@ length = len(r) if do_unicode and isinstance(r, str): # convert string to unicode explicitely here - r = unicode(r) + from pypy.objspace.std.unicodetype import plain_str2unicode + r = plain_str2unicode(self.space, r) prec = self.prec if prec == -1 and self.width == 0: # fast path @@ -488,7 +489,8 @@ result = formatter.format() except NeedUnicodeFormattingError: # fall through to the unicode case - fmt = unicode(fmt) + from pypy.objspace.std.unicodetype import plain_str2unicode + fmt = plain_str2unicode(space, fmt) else: return space.wrap(result) else: Modified: pypy/branch/cpython-extension/pypy/objspace/std/frame.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/frame.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/frame.py Fri Apr 23 02:41:35 2010 @@ -3,17 +3,20 @@ import operator from pypy.rlib.unroll import unrolling_iterable -from pypy.interpreter import pyframe, pyopcode, function +from pypy.interpreter import pyopcode, function +from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module.__builtin__ import OPTIMIZED_BUILTINS, Module +from pypy.objspace.std import intobject, smallintobject from pypy.objspace.std.multimethod import FailedToImplement +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.listobject import W_ListObject -class BaseFrame(pyframe.PyFrame): +class BaseFrame(PyFrame): """These opcodes are always overridden.""" def LIST_APPEND(f, oparg, next_instr): - from pypy.objspace.std.listobject import W_ListObject w = f.popvalue() v = f.popvalue() if type(v) is W_ListObject: @@ -23,13 +26,12 @@ def small_int_BINARY_ADD(f, oparg, next_instr): - from pypy.objspace.std.smallintobject import (W_SmallIntObject, - add__SmallInt_SmallInt) w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_SmallIntObject and type(w_2) is W_SmallIntObject: + if (type(w_1) is smallintobject.W_SmallIntObject and + type(w_2) is smallintobject.W_SmallIntObject): try: - w_result = add__SmallInt_SmallInt(f.space, w_1, w_2) + w_result = smallintobject.add__SmallInt_SmallInt(f.space, w_1, w_2) except FailedToImplement: w_result = f.space.add(w_1, w_2) else: @@ -38,12 +40,12 @@ def int_BINARY_ADD(f, oparg, next_instr): - from pypy.objspace.std.intobject import W_IntObject, add__Int_Int w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_IntObject and type(w_2) is W_IntObject: + if (type(w_1) is intobject.W_IntObject and + type(w_2) is intobject.W_IntObject): try: - w_result = add__Int_Int(f.space, w_1, w_2) + w_result = intobject.add__Int_Int(f.space, w_1, w_2) except FailedToImplement: w_result = f.space.add(w_1, w_2) else: @@ -52,11 +54,9 @@ def list_BINARY_SUBSCR(f, oparg, next_instr): - from pypy.objspace.std.intobject import W_IntObject - from pypy.objspace.std.listobject import W_ListObject w_2 = f.popvalue() w_1 = f.popvalue() - if type(w_1) is W_ListObject and type(w_2) is W_IntObject: + if type(w_1) is W_ListObject and type(w_2) is intobject.W_IntObject: try: w_result = w_1.wrappeditems[w_2.intval] except IndexError: @@ -67,7 +67,6 @@ f.pushvalue(w_result) def CALL_LIKELY_BUILTIN(f, oparg, next_instr): - from pypy.objspace.std.dictmultiobject import W_DictMultiObject w_globals = f.w_globals num = oparg >> 8 assert isinstance(w_globals, W_DictMultiObject) @@ -75,7 +74,7 @@ if w_value is None: builtins = f.get_builtin() assert isinstance(builtins, Module) - w_builtin_dict = builtins.w_dict + w_builtin_dict = builtins.getdict() assert isinstance(w_builtin_dict, W_DictMultiObject) w_value = w_builtin_dict.get_builtin_indexed(num) if w_value is None: @@ -113,12 +112,12 @@ unrolling_compare_ops = unrolling_iterable(enumerate(compare_table)) def fast_COMPARE_OP(f, testnum, next_instr): - from pypy.objspace.std.intobject import W_IntObject w_2 = f.popvalue() w_1 = f.popvalue() w_result = None - if (type(w_2) is W_IntObject and type(w_1) is W_IntObject - and testnum < len(compare_table)): + if (type(w_2) is intobject.W_IntObject and + type(w_1) is intobject.W_IntObject and + testnum < len(compare_table)): for i, attr in unrolling_compare_ops: if i == testnum: op = getattr(operator, attr) Modified: pypy/branch/cpython-extension/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/intobject.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/intobject.py Fri Apr 23 02:41:35 2010 @@ -34,6 +34,10 @@ registerimplementation(W_IntObject) +# NB: This code is shared by smallintobject.py, and thus no other Int +# multimethods should be invoked from these implementations. Instead, add an +# alias and then teach copy_multimethods in smallintobject.py to override +# it. See int__Int for example. def int_w__Int(space, w_int1): return int(w_int1.intval) @@ -56,8 +60,7 @@ str__Int = repr__Int -def declare_new_int_comparison(opname, clsname): - # also used by smallintobject.py +def declare_new_int_comparison(opname): import operator from pypy.tool.sourcetools import func_with_new_name op = getattr(operator, opname) @@ -65,18 +68,18 @@ i = w_int1.intval j = w_int2.intval return space.newbool(op(i, j)) - name = "%s__%s_%s" % (opname, clsname, clsname) + name = "%s__Int_Int" % (opname,) return func_with_new_name(f, name), name for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op, "Int") + func, name = declare_new_int_comparison(op) globals()[name] = func def hash__Int(space, w_int1): # unlike CPython, we don't special-case the value -1 in most of our # hash functions, so there is not much sense special-casing it here either. # Make sure this is consistent with the hash of floats and longs. - return int__Int(space, w_int1) + return get_integer(space, w_int1) # coerce def coerce__Int_Int(space, w_int1, w_int2): @@ -217,13 +220,14 @@ raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer negation")) return wrapint(space, x) +get_negint = neg__Int def abs__Int(space, w_int1): if w_int1.intval >= 0: - return pos__Int(space, w_int1) + return get_integer(space, w_int1) else: - return neg__Int(space, w_int1) + return get_negint(space, w_int1) def nonzero__Int(space, w_int1): return space.newbool(w_int1.intval != 0) @@ -240,7 +244,7 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return get_integer(space, w_int1) if b >= LONG_BIT: raise FailedToImplementArgs(space.w_OverflowError, space.wrap("integer left shift")) @@ -258,7 +262,7 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return get_integer(space, w_int1) if b >= LONG_BIT: if a < 0: a = -1 @@ -294,10 +298,11 @@ return w_int1 a = w_int1.intval return wrapint(space, a) +get_integer = int__Int pos__Int = int__Int def index__Int(space, w_int1): - return int__Int(space, w_int1) + return get_integer(space, w_int1) def float__Int(space, w_int1): a = w_int1.intval Modified: pypy/branch/cpython-extension/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/objspace.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/objspace.py Fri Apr 23 02:41:35 2010 @@ -1,7 +1,7 @@ import __builtin__ import types from pypy.interpreter import pyframe, function, special -from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, UnpackValueError +from pypy.interpreter.baseobjspace import ObjSpace, Wrappable from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.typedef import get_unique_interplevel_subclass from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model, @@ -133,7 +133,7 @@ if not we_are_translated() and isinstance(code, CPythonFakeCode): return CPythonFakeFrame(self, code, w_globals) else: - return self.FrameClass(self, code, w_globals, closure) + return ObjSpace.createframe(self, code, w_globals, closure) def gettypefor(self, cls): return self.gettypeobject(cls.typedef) @@ -252,9 +252,6 @@ raise model.UnwrapError, "cannot unwrap: %r" % w_obj def newint(self, intval): - # this time-critical and circular-imports-funny method was stored - # on 'self' by initialize() - # not sure how bad this is: return wrapint(self, intval) def newfloat(self, floatval): @@ -338,6 +335,10 @@ # have different return type. First one is a resizable list, second # one is not + def _wrap_expected_length(self, expected, got): + return OperationError(self.w_ValueError, + self.wrap("Expected length %d, got %d" % (expected, got))) + def unpackiterable(self, w_obj, expected_length=-1): if isinstance(w_obj, W_TupleObject): t = w_obj.wrappeditems[:] @@ -346,7 +347,7 @@ else: return ObjSpace.unpackiterable(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def fixedview(self, w_obj, expected_length=-1): @@ -359,7 +360,7 @@ else: return ObjSpace.fixedview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def listview(self, w_obj, expected_length=-1): @@ -370,7 +371,7 @@ else: return ObjSpace.listview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + raise self._wrap_expected_length(expected_length, len(t)) return t def sliceindices(self, w_slice, w_length): @@ -465,11 +466,11 @@ return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) - def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + def setitem_str(self, w_obj, key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): - w_obj.set_str_keyed_item(key, w_value, shadows_type) + w_obj.setitem_str(key, w_value, shadows_type) else: self.setitem(w_obj, self.wrap(key), w_value) Modified: pypy/branch/cpython-extension/pypy/objspace/std/smallintobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/smallintobject.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/smallintobject.py Fri Apr 23 02:41:35 2010 @@ -2,18 +2,18 @@ Implementation of small ints, stored as odd-valued pointers in the translated PyPy. To enable them, see inttype.py. """ +import types from pypy.interpreter.error import OperationError +from pypy.objspace.std import intobject from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all from pypy.objspace.std.multimethod import FailedToImplementArgs from pypy.objspace.std.noneobject import W_NoneObject from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.intobject import W_IntObject, declare_new_int_comparison +from pypy.objspace.std.intobject import W_IntObject, _impl_int_int_pow from pypy.rlib.objectmodel import UnboxedValue - -# XXX this is a complete copy of intobject.py. Find a better but still -# XXX annotator-friendly way to share code... +from pypy.rlib.rbigint import rbigint class W_SmallIntObject(W_Object, UnboxedValue): @@ -39,215 +39,19 @@ def delegate_SmallInt2Complex(space, w_small): return space.newcomplex(float(w_small.intval), 0.0) +def copy_multimethods(ns): + """Copy integer multimethods for small int.""" + for name, func in intobject.__dict__.iteritems(): + if "__Int" in name: + new_name = name.replace("Int", "SmallInt") + # Copy the function, so the annotator specializes it for + # W_SmallIntObject. + ns[new_name] = types.FunctionType(func.func_code, ns, new_name, + func.func_defaults, + func.func_closure) + ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"] + ns["get_negint"] = ns["neg__SmallInt"] -def int_w__SmallInt(space, w_int1): - return int(w_int1.intval) - -def uint_w__SmallInt(space, w_int1): - intval = w_int1.intval - if intval < 0: - raise OperationError(space.w_ValueError, - space.wrap("cannot convert negative integer to unsigned")) - else: - return r_uint(intval) - -def repr__SmallInt(space, w_int1): - a = w_int1.intval - res = str(a) - return space.wrap(res) - -str__SmallInt = repr__SmallInt - -for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op, "SmallInt") - globals()[name] = func - -def hash__SmallInt(space, w_int1): - # unlike CPython, we don't special-case the value -1 in most of our - # hash functions, so there is not much sense special-casing it here either. - # Make sure this is consistent with the hash of floats and longs. - return int__SmallInt(space, w_int1) - -# coerce -def coerce__SmallInt_SmallInt(space, w_int1, w_int2): - return space.newtuple([w_int1, w_int2]) - - -def add__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - # note that no overflow checking is necessary here: x and y fit into 31 - # bits (or 63 bits respectively), so their sum fits into 32 (or 64) bits. - # wrapint then makes sure that either a tagged int or a normal int is - # created - return wrapint(space, x + y) - -def sub__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - # see comment in add__SmallInt_SmallInt - return wrapint(space, x - y) - -def mul__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - try: - z = ovfcheck(x * y) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer multiplication")) - return wrapint(space, z) - -def div__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer division by zero")) - # no overflow possible - return wrapint(space, x // y) - -floordiv__SmallInt_SmallInt = div__SmallInt_SmallInt - -def truediv__SmallInt_SmallInt(space, w_int1, w_int2): - x = float(w_int1.intval) - y = float(w_int2.intval) - if y == 0.0: - raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division")) - return space.wrap(x / y) - -def mod__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer modulo by zero")) - # no overflow possible - return wrapint(space, x % y) - -def divmod__SmallInt_SmallInt(space, w_int1, w_int2): - x = w_int1.intval - y = w_int2.intval - if y == 0: - raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer divmod by zero")) - # no overflow possible - z = x // y - m = x % y - return space.newtuple([space.wrap(z), space.wrap(m)]) - -def pow__SmallInt_SmallInt_SmallInt(space, w_int1, w_int2, w_int3): - from pypy.objspace.std.intobject import _impl_int_int_pow - x = w_int1.intval - y = w_int2.intval - z = w_int3.intval - if z == 0: - raise OperationError(space.w_ValueError, - space.wrap("pow() 3rd argument cannot be 0")) - return _impl_int_int_pow(space, x, y, z) - -def pow__SmallInt_SmallInt_None(space, w_int1, w_int2, w_int3): - from pypy.objspace.std.intobject import _impl_int_int_pow - x = w_int1.intval - y = w_int2.intval - return _impl_int_int_pow(space, x, y) - -def neg__SmallInt(space, w_int1): - a = w_int1.intval - # no overflow possible since a fits into 31/63 bits - return wrapint(space, -a) - - -def abs__SmallInt(space, w_int1): - if w_int1.intval >= 0: - return pos__SmallInt(space, w_int1) - else: - return neg__SmallInt(space, w_int1) - -def nonzero__SmallInt(space, w_int1): - return space.newbool(w_int1.intval != 0) - -def invert__SmallInt(space, w_int1): - x = w_int1.intval - a = ~x - return wrapint(space, a) - -def lshift__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return int__SmallInt(space, w_int1) - if b >= LONG_BIT: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - try: - c = ovfcheck_lshift(a, b) - except OverflowError: - raise FailedToImplementArgs(space.w_OverflowError, - space.wrap("integer left shift")) - return wrapint(space, c) - -def rshift__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - if b < 0: - raise OperationError(space.w_ValueError, - space.wrap("negative shift count")) - if a == 0 or b == 0: - return int__SmallInt(space, w_int1) - if b >= LONG_BIT: - if a < 0: - a = -1 - else: - a = 0 - else: - a = a >> b - return wrapint(space, a) - -def and__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a & b - return wrapint(space, res) - -def xor__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a ^ b - return wrapint(space, res) - -def or__SmallInt_SmallInt(space, w_int1, w_int2): - a = w_int1.intval - b = w_int2.intval - res = a | b - return wrapint(space, res) - -# int__SmallInt is supposed to do nothing, unless it has -# a derived integer object, where it should return -# an exact one. -def int__SmallInt(space, w_int1): - if space.is_w(space.type(w_int1), space.w_int): - return w_int1 - a = w_int1.intval - return W_SmallIntObject(a) -pos__SmallInt = int__SmallInt - -def float__SmallInt(space, w_int1): - a = w_int1.intval - x = float(a) - return space.newfloat(x) - -def oct__SmallInt(space, w_int1): - return space.wrap(oct(w_int1.intval)) - -def hex__SmallInt(space, w_int1): - return space.wrap(hex(w_int1.intval)) - -def getnewargs__SmallInt(space, w_int1): - return space.newtuple([wrapint(space, w_int1.intval)]) - +copy_multimethods(globals()) register_all(vars()) Modified: pypy/branch/cpython-extension/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/stringobject.py Fri Apr 23 02:41:35 2010 @@ -39,25 +39,10 @@ W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)] del i -def _decode_ascii(space, s): - try: - return s.decode("ascii") - except UnicodeDecodeError: - for i in range(len(s)): - if ord(s[i]) > 127: - raise OperationError( - space.w_UnicodeDecodeError, - space.newtuple([ - space.wrap('ascii'), - space.wrap(s), - space.wrap(i), - space.wrap(i+1), - space.wrap("ordinal not in range(128)")])) - assert False, "unreachable" - def unicode_w__String(space, w_self): # XXX should this use the default encoding? - return _decode_ascii(space, w_self._value) + from pypy.objspace.std.unicodetype import plain_str2unicode + return plain_str2unicode(space, w_self._value) def _is_generic(space, w_self, fun): v = w_self._value Modified: pypy/branch/cpython-extension/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/test/test_obj.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/test/test_obj.py Fri Apr 23 02:41:35 2010 @@ -7,11 +7,14 @@ cpython_behavior = (not option.runappdirect or not hasattr(sys, 'pypy_translation_info')) - cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_behavior = cls.space.wrap(cpython_behavior) + cls.w_cpython_version = cls.space.wrap(tuple(sys.version_info)) def test_hash_builtin(self): if not self.cpython_behavior: skip("on pypy-c id == hash is not guaranteed") + if self.cpython_version >= (2, 7): + skip("on CPython >= 2.7, id != hash") import sys o = object() assert (hash(o) & sys.maxint) == (id(o) & sys.maxint) @@ -33,7 +36,7 @@ class X(object): pass x = X() - if self.cpython_behavior: + if self.cpython_behavior and self.cpython_version < (2, 7): assert (hash(x) & sys.maxint) == (id(x) & sys.maxint) assert hash(x) == object.__hash__(x) Modified: pypy/branch/cpython-extension/pypy/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/test/test_stringformat.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/test/test_stringformat.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,4 @@ +# coding: utf-8 class AppTestStringObjectWithDict: @@ -170,6 +171,9 @@ raises(TypeError, '%c'.__mod__, ("bla",)) raises(TypeError, '%c'.__mod__, ("",)) raises(TypeError, '%c'.__mod__, (['c'],)) + + def test_broken_unicode(self): + raises(UnicodeDecodeError, 'N?zov: %s'.__mod__, u'Jerry') class AppTestWidthPrec: def test_width(self): Modified: pypy/branch/cpython-extension/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/cpython-extension/pypy/objspace/std/unicodetype.py Fri Apr 23 02:41:35 2010 @@ -13,6 +13,22 @@ return wrapunicode(space, uni) return W_UnicodeObject(uni) +def plain_str2unicode(space, s): + try: + return unicode(s) + except UnicodeDecodeError: + for i in range(len(s)): + if ord(s[i]) > 127: + raise OperationError( + space.w_UnicodeDecodeError, + space.newtuple([ + space.wrap('ascii'), + space.wrap(s), + space.wrap(i), + space.wrap(i+1), + space.wrap("ordinal not in range(128)")])) + assert False, "unreachable" + unicode_capitalize = SMM('capitalize', 1, doc='S.capitalize() -> unicode\n\nReturn a' Modified: pypy/branch/cpython-extension/pypy/rlib/nonconst.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/nonconst.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/nonconst.py Fri Apr 23 02:41:35 2010 @@ -3,9 +3,7 @@ """ from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rpython.lltypesystem import lltype +from pypy.objspace.flow.model import Constant class NonConstant(object): def __init__(self, _constant): @@ -22,7 +20,7 @@ class EntryNonConstant(ExtRegistryEntry): _about_ = NonConstant - + def compute_result_annotation(self, arg): if hasattr(arg, 'const'): return self.bookkeeper.immutablevalue(arg.const, False) Modified: pypy/branch/cpython-extension/pypy/rlib/rsre/rsre_char.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rlib/rsre/rsre_char.py (original) +++ pypy/branch/cpython-extension/pypy/rlib/rsre/rsre_char.py Fri Apr 23 02:41:35 2010 @@ -2,7 +2,7 @@ Character categories and charsets. """ import sys -from pypy.rlib.rsre._rsre_platform import tolower, isalnum +from pypy.rlib.rlocale import tolower, isalnum from pypy.rlib.unroll import unrolling_iterable # Note: the unicode parts of this module require you to call Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/lltype.py Fri Apr 23 02:41:35 2010 @@ -3,9 +3,9 @@ import sys import py -from pypy.rlib.rarithmetic import r_int, r_uint, intmask, r_singlefloat -from pypy.rlib.rarithmetic import r_ulonglong, r_longlong, base_int -from pypy.rlib.rarithmetic import normalizedinttype +from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, + r_ulonglong, r_longlong, base_int, + normalizedinttype) from pypy.rlib.objectmodel import Symbolic from pypy.tool.uid import Hashable from pypy.tool.tls import tlsobject @@ -15,8 +15,6 @@ import struct import weakref -log = py.log.Producer('lltype') - TLS = tlsobject() TRACK_ALLOCATIONS = False Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/rffi.py Fri Apr 23 02:41:35 2010 @@ -19,9 +19,6 @@ from pypy.rpython.lltypesystem import llmemory import os, sys -class UnhandledRPythonException(Exception): - pass - class CConstant(Symbolic): """ A C-level constant, maybe #define, rendered directly. """ Modified: pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/lltypesystem/test/test_lltype.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.lib.identity_dict import identity_dict Modified: pypy/branch/cpython-extension/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/memory/gctransform/framework.py Fri Apr 23 02:41:35 2010 @@ -445,11 +445,7 @@ self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) self.malloc_zero_filled = GCClass.malloc_zero_filled - HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR - self._gc_fields = fields = [] - for fldname in HDR._names: - FLDTYPE = getattr(HDR, fldname) - fields.append(('_' + fldname, FLDTYPE)) + HDR = self.HDR = self.gcdata.gc.gcheaderbuilder.HDR size_gc_header = self.gcdata.gc.gcheaderbuilder.size_gc_header vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field) @@ -472,26 +468,20 @@ def finalizer_funcptr_for_type(self, TYPE): return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) - def gc_fields(self): - return self._gc_fields - - def gc_field_values_for(self, obj, needs_hash=False): + def gc_header_for(self, obj, needs_hash=False): hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj) - HDR = self._gc_HDR + HDR = self.HDR withhash, flag = self.gcdata.gc.withhash_flag_is_in_field - result = [] - for fldname in HDR._names: - x = getattr(hdr, fldname) - if fldname == withhash: - TYPE = lltype.typeOf(x) - x = lltype.cast_primitive(lltype.Signed, x) - if needs_hash: - x |= flag # set the flag in the header - else: - x &= ~flag # clear the flag in the header - x = lltype.cast_primitive(TYPE, x) - result.append(x) - return result + x = getattr(hdr, withhash) + TYPE = lltype.typeOf(x) + x = lltype.cast_primitive(lltype.Signed, x) + if needs_hash: + x |= flag # set the flag in the header + else: + x &= ~flag # clear the flag in the header + x = lltype.cast_primitive(TYPE, x) + setattr(hdr, withhash, x) + return hdr def get_hash_offset(self, T): type_id = self.get_type_id(T) Modified: pypy/branch/cpython-extension/pypy/rpython/rint.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/rint.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/rint.py Fri Apr 23 02:41:35 2010 @@ -1,7 +1,7 @@ import sys from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel -from pypy.objspace.flow.objspace import op_appendices +from pypy.objspace.flow.operation import op_appendices from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Bool, Float, \ Void, Char, UniChar, malloc, pyobjectptr, UnsignedLongLong, \ SignedLongLong, build_number, Number, cast_primitive, typeOf Modified: pypy/branch/cpython-extension/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/test/test_llann.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/test/test_llann.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECTPTR Modified: pypy/branch/cpython-extension/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/test/test_rlist.py Fri Apr 23 02:41:35 2010 @@ -1,4 +1,6 @@ import sys +import re +import py from pypy.translator.translator import TranslationContext from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem.lltype import * @@ -11,7 +13,6 @@ from pypy.translator.translator import TranslationContext from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -import re # undo the specialization parameter for n1 in 'get set del'.split(): Modified: pypy/branch/cpython-extension/pypy/rpython/test/test_rpbc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/rpython/test/test_rpbc.py (original) +++ pypy/branch/cpython-extension/pypy/rpython/test/test_rpbc.py Fri Apr 23 02:41:35 2010 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.rtyper import RPythonTyper from pypy.rpython.ootypesystem import ootype Modified: pypy/branch/cpython-extension/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/cpython-extension/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/cpython-extension/pypy/tool/pytest/appsupport.py Fri Apr 23 02:41:35 2010 @@ -87,7 +87,7 @@ if debug_excs: self._excinfo = debug_excs[0] - def exconly(self, tryshort=True): + def exconly(self, tryshort=True): return '(application-level) ' + self.operr.errorstr(self.space) def errisinstance(self, exc): @@ -218,7 +218,8 @@ for key, w_value in kwds_w.items(): space.setitem(w_locals, space.wrap(key), w_value) try: - space.exec_(source.compile(), frame.w_globals, w_locals) + space.exec_(str(source), frame.w_globals, w_locals, + filename=__file__) except OperationError, e: if e.match(space, w_ExpectedException): return _exc_info(space, e) Modified: pypy/branch/cpython-extension/pypy/tool/sourcetools.py ============================================================================== --- pypy/branch/cpython-extension/pypy/tool/sourcetools.py (original) +++ pypy/branch/cpython-extension/pypy/tool/sourcetools.py Fri Apr 23 02:41:35 2010 @@ -17,7 +17,7 @@ indentation. The shorter triple quotes are choosen automatically. The result is returned as a 1-tuple.""" - if type(func) is not str: + if not isinstance(func, str): doc = func.__doc__ else: doc = func @@ -217,20 +217,15 @@ # ____________________________________________________________ -if sys.version_info >= (2, 3): - def func_with_new_name(func, newname): - """Make a renamed copy of a function.""" - f = new.function(func.func_code, func.func_globals, - newname, func.func_defaults, - func.func_closure) - if func.func_dict: - f.func_dict = {} - f.func_dict.update(func.func_dict) - return f -else: - raise Exception("sorry, Python 2.2 not supported") - # because we need to return a new function object -- impossible in 2.2, - # cannot create functions with closures without using veeeery strange code +def func_with_new_name(func, newname): + """Make a renamed copy of a function.""" + f = new.function(func.func_code, func.func_globals, + newname, func.func_defaults, + func.func_closure) + if func.func_dict: + f.func_dict = {} + f.func_dict.update(func.func_dict) + return f PY_IDENTIFIER = ''.join([(('0' <= chr(i) <= '9' or 'a' <= chr(i) <= 'z' or Modified: pypy/branch/cpython-extension/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/cpython-extension/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/cpython-extension/pypy/tool/stdlib_opcode.py Fri Apr 23 02:41:35 2010 @@ -1,44 +1,25 @@ +""" +Opcodes PyPy compiles Python source to. +Also gives access to opcodes of the host Python PyPy was bootstrapped with +(module attributes with the `host_` prefix). +""" + # load opcode.py as pythonopcode from our own lib __all__ = ['opmap', 'opname', 'HAVE_ARGUMENT', 'hasconst', 'hasname', 'hasjrel', 'hasjabs', 'haslocal', 'hascompare', 'hasfree', 'cmp_op'] -def load_opcode(): - import py - opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') - d = {} - execfile(str(opcode_path), d) - return d - -opcode_dict = load_opcode() -del load_opcode - -# copy some stuff from opcode.py directly into our globals -for name in __all__: - if name in opcode_dict: - globals()[name] = opcode_dict[name] -globals().update(opmap) -SLICE = opmap["SLICE+0"] -STORE_SLICE = opmap["STORE_SLICE+0"] -DELETE_SLICE = opmap["DELETE_SLICE+0"] - -opcode_method_names = ['MISSING_OPCODE'] * 256 -for name, index in opmap.items(): - opcode_method_names[index] = name.replace('+', '_') - # ____________________________________________________________ # RPython-friendly helpers and structures -from pypy.rlib.unroll import unrolling_iterable - - -class OpcodeDesc(object): - def __init__(self, name, index): +class _BaseOpcodeDesc(object): + def __init__(self, bytecode_spec, name, index, methodname): + self.bytecode_spec = bytecode_spec self.name = name - self.methodname = opcode_method_names[index] + self.methodname = methodname self.index = index - self.hasarg = index >= HAVE_ARGUMENT + self.hasarg = index >= self.HAVE_ARGUMENT def _freeze_(self): return True @@ -64,24 +45,87 @@ return (cmp(self.__class__, other.__class__) or cmp(self.sortkey(), other.sortkey())) -opdescmap = {} + def __str__(self): + return "" % (self.index, self.name, id(self)) + + __repr__ = __str__ + +class _baseopcodedesc: + """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs.""" + pass + + +class BytecodeSpec(object): + """A bunch of mappings describing a bytecode instruction set.""" + + def __init__(self, name, opmap, HAVE_ARGUMENT): + """NOT_RPYTHON.""" + class OpcodeDesc(_BaseOpcodeDesc): + HAVE_ARGUMENT = HAVE_ARGUMENT + class opcodedesc(_baseopcodedesc): + """A namespace mapping OPCODE_NAME to OpcodeDescs.""" + + self.name = name + self.OpcodeDesc = OpcodeDesc + self.opcodedesc = opcodedesc + self.HAVE_ARGUMENT = HAVE_ARGUMENT + # opname -> opcode + self.opmap = opmap + # opcode -> method name + self.method_names = tbl = ['MISSING_OPCODE'] * 256 + # opcode -> opdesc + self.opdescmap = {} + for name, index in opmap.items(): + tbl[index] = methodname = name.replace('+', '_') + desc = OpcodeDesc(self, name, index, methodname) + setattr(self.opcodedesc, name, desc) + self.opdescmap[index] = desc + # fill the ordered opdesc list + self.ordered_opdescs = lst = self.opdescmap.values() + lst.sort() + + def to_globals(self): + """NOT_RPYTHON. Add individual opcodes to the module constants.""" + g = globals() + g.update(self.opmap) + g['SLICE'] = self.opmap["SLICE+0"] + g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] + g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + + def __str__(self): + return "<%s bytecode>" % (self.name,) + + __repr__ = __str__ + + +# Initialization + +from pypy.rlib.unroll import unrolling_iterable + +from opcode import ( + opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) -class opcodedesc: - """A namespace mapping OPCODE_NAME to OpcodeDescs.""" +def load_pypy_opcode(): + import py + opcode_path = py.path.local(__file__).dirpath().dirpath().dirpath('lib-python/modified-2.5.2/opcode.py') + d = {} + execfile(str(opcode_path), d) + for name in __all__: + if name in d: + globals()[name] = d[name] + return d -for name, index in opmap.items(): - desc = OpcodeDesc(name, index) - setattr(opcodedesc, name, desc) - opdescmap[index] = desc - -lst = opdescmap.values() -lst.sort() -unrolling_opcode_descs = unrolling_iterable(lst) - -# Allow non-translated code to interpret the new 2.6 bytecodes -import sys -if sys.version_info >= (2, 6): - import opcode - opcode_method_names[opcode.opmap['STORE_MAP']] = 'STORE_MAP' +load_pypy_opcode() +del load_pypy_opcode -del name, index, desc, lst +bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT) +host_bytecode_spec = BytecodeSpec('host', host_opmap, host_HAVE_ARGUMENT) +bytecode_spec.to_globals() + +opcode_method_names = bytecode_spec.method_names +opcodedesc = bytecode_spec.opcodedesc + +unrolling_all_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs + host_bytecode_spec.ordered_opdescs) +unrolling_opcode_descs = unrolling_iterable( + bytecode_spec.ordered_opdescs) Modified: pypy/branch/cpython-extension/pypy/translator/c/database.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/database.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/database.py Fri Apr 23 02:41:35 2010 @@ -177,58 +177,56 @@ return node def get(self, obj): - # XXX extra indent is preserve svn blame - kind of important IMHO (rxe) - if 1: - if isinstance(obj, CConstant): - return obj.c_name # without further checks - T = typeOf(obj) - if isinstance(T, Primitive) or T == GCREF: - return PrimitiveName[T](obj, self) - elif isinstance(T, Ptr): - if obj: # test if the ptr is non-NULL - try: - container = obj._obj - except lltype.DelayedPointer: - # hack hack hack - name = obj._obj0 - assert name.startswith('delayed!') - n = len('delayed!') - if len(name) == n: - raise - if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): - if obj in self.idelayedfunctionnames: - return self.idelayedfunctionnames[obj][0] - funcname = name[n:] - funcname = self.namespace.uniquename('g_'+funcname) - self.idelayedfunctionnames[obj] = funcname, obj - else: - funcname = None # can't use the name of a - # delayed non-function ptr - self.delayedfunctionptrs.append(obj) - return funcname - # /hack hack hack - else: - # hack hack hack + if isinstance(obj, CConstant): + return obj.c_name # without further checks + T = typeOf(obj) + if isinstance(T, Primitive) or T == GCREF: + return PrimitiveName[T](obj, self) + elif isinstance(T, Ptr): + if obj: # test if the ptr is non-NULL + try: + container = obj._obj + except lltype.DelayedPointer: + # hack hack hack + name = obj._obj0 + assert name.startswith('delayed!') + n = len('delayed!') + if len(name) == n: + raise + if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): if obj in self.idelayedfunctionnames: - # this used to be a delayed function, - # make sure we use the same name - forcename = self.idelayedfunctionnames[obj][0] - node = self.getcontainernode(container, - forcename=forcename) - assert node.ptrname == forcename - return forcename - # /hack hack hack - - if isinstance(container, int): - # special case for tagged odd-valued pointers - return '((%s) %d)' % (cdecl(self.gettype(T), ''), - obj._obj) - node = self.getcontainernode(container) - return node.ptrname + return self.idelayedfunctionnames[obj][0] + funcname = name[n:] + funcname = self.namespace.uniquename('g_'+funcname) + self.idelayedfunctionnames[obj] = funcname, obj + else: + funcname = None # can't use the name of a + # delayed non-function ptr + self.delayedfunctionptrs.append(obj) + return funcname + # /hack hack hack else: - return '((%s) NULL)' % (cdecl(self.gettype(T), ''), ) + # hack hack hack + if obj in self.idelayedfunctionnames: + # this used to be a delayed function, + # make sure we use the same name + forcename = self.idelayedfunctionnames[obj][0] + node = self.getcontainernode(container, + forcename=forcename) + assert node.ptrname == forcename + return forcename + # /hack hack hack + + if isinstance(container, int): + # special case for tagged odd-valued pointers + return '((%s) %d)' % (cdecl(self.gettype(T), ''), + obj._obj) + node = self.getcontainernode(container) + return node.ptrname else: - raise Exception("don't know about %r" % (obj,)) + return '((%s) NULL)' % (cdecl(self.gettype(T), ''), ) + else: + raise Exception("don't know about %r" % (obj,)) def complete(self, show_progress=True): assert not self.completed @@ -304,7 +302,7 @@ self.getcontainernode(value) else: self.get(value) - + while True: while True: while self.pendingsetupnodes: Modified: pypy/branch/cpython-extension/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/gc.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/gc.py Fri Apr 23 02:41:35 2010 @@ -20,16 +20,13 @@ def common_gcheader_definition(self, defnode): if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR - return [(name, HDR._flds[name]) for name in HDR._names] - else: - return [] + return defnode.db.gctransformer.HDR + return None def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: raise NotImplementedError - else: - return [] + return None def struct_gcheader_definition(self, defnode): return self.common_gcheader_definition(defnode) @@ -113,11 +110,9 @@ def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: gct = defnode.db.gctransformer - hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) - HDR = gct.HDR - return [getattr(hdr, fldname) for fldname in HDR._names] - else: - return [] + top = top_container(defnode.obj) + return gct.gcheaderbuilder.header_of_object(top)._obj + return None # for structs @@ -196,9 +191,10 @@ def common_gcheader_initdata(self, defnode): if defnode.db.gctransformer is not None: - return [lltype.identityhash_nocache(defnode.obj._as_ptr())] - else: - return [] + hdr = lltype.malloc(defnode.db.gctransformer.HDR, immortal=True) + hdr.hash = lltype.identityhash_nocache(defnode.obj._as_ptr()) + return hdr._obj + return None def array_setup(self, arraydefnode): pass @@ -333,13 +329,11 @@ args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) - def common_gcheader_definition(self, defnode): - return defnode.db.gctransformer.gc_fields() - def common_gcheader_initdata(self, defnode): o = top_container(defnode.obj) needs_hash = self.get_prebuilt_hash(o) is not None - return defnode.db.gctransformer.gc_field_values_for(o, needs_hash) + hdr = defnode.db.gctransformer.gc_header_for(o, needs_hash) + return hdr._obj def get_prebuilt_hash(self, obj): # for prebuilt objects that need to have their hash stored and @@ -362,9 +356,14 @@ # all implemented by a single call to a C macro. [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args typename = funcgen.db.gettype(op.result.concretetype) - fieldname = c_vtableinfo.value[2] + tid_field = c_vtableinfo.value[2] + # Fish out the C name of the tid field. + HDR = self.db.gctransformer.HDR + hdr_node = self.db.gettypedefnode(HDR) + fieldname = hdr_node.c_struct_field_name(tid_field) return ( - '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s->_%s, %s);' + '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s->' + '_gcheader.%s, %s);' % (funcgen.expr(op.result), cdecl(typename, ''), funcgen.expr(c_grpptr), Modified: pypy/branch/cpython-extension/pypy/translator/c/node.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/node.py (original) +++ pypy/branch/cpython-extension/pypy/translator/c/node.py Fri Apr 23 02:41:35 2010 @@ -89,8 +89,10 @@ if self.varlength != 1: self.normalizedtypename = db.gettype(STRUCT, who_asks=self) if needs_gcheader(self.STRUCT): - for fname, T in db.gcpolicy.struct_gcheader_definition(self): - self.fields.append((fname, db.gettype(T, who_asks=self))) + HDR = db.gcpolicy.struct_gcheader_definition(self) + if HDR is not None: + gc_field = ("_gcheader", db.gettype(HDR, who_asks=self)) + self.fields.append(gc_field) for name in self.fieldnames: T = self.c_struct_field_type(name) if name == STRUCT._arrayfld: @@ -219,8 +221,10 @@ if self.varlength != 1: self.normalizedtypename = db.gettype(ARRAY, who_asks=self) if needs_gcheader(ARRAY): - for fname, T in db.gcpolicy.array_gcheader_definition(self): - self.gcfields.append((fname, db.gettype(T, who_asks=self))) + HDR = db.gcpolicy.array_gcheader_definition(self) + if HDR is not None: + gc_field = ("_gcheader", db.gettype(HDR, who_asks=self)) + self.gcfields.append(gc_field) self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) def computegcinfo(self): @@ -563,12 +567,12 @@ data = [] if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.struct_gcheader_initdata(self)): - data.append(('gcheader%d'%i, thing)) - + gc_init = self.db.gcpolicy.struct_gcheader_initdata(self) + data.append(('gcheader', gc_init)) + for name in defnode.fieldnames: data.append((name, getattr(self.obj, name))) - + # Reasonably, you should only initialise one of the fields of a union # in C. This is possible with the syntax '.fieldname value' or # '.fieldname = value'. But here we don't know which of the @@ -670,12 +674,11 @@ defnode = self.db.gettypedefnode(self.T) yield '{' if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.array_gcheader_initdata(self)): - lines = generic_initializationexpr(self.db, thing, - 'gcheader%d'%i, - '%sgcheader%d' % (decoration, i)) - for line in lines: - yield line + gc_init = self.db.gcpolicy.array_gcheader_initdata(self) + lines = generic_initializationexpr(self.db, gc_init, 'gcheader', + '%sgcheader' % (decoration,)) + for line in lines: + yield line if self.T._hints.get('nolength', False): length = '' else: Modified: pypy/branch/cpython-extension/pypy/translator/c/src/signals.h ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/c/src/signals.h (original) +++ pypy/branch/cpython-extension/pypy/translator/c/src/signals.h Fri Apr 23 02:41:35 2010 @@ -70,7 +70,11 @@ cleared again. The variable is exposed and RPython code is free to use the other bits in any way. */ #define PENDING_SIGNAL_BIT (LONG_MIN) /* high bit */ -extern long pypysig_occurred; +/* This is a struct for the JIT. See interp_signal.py. */ +struct pypysig_long_struct { + long value; +}; +extern struct pypysig_long_struct pypysig_occurred; /* some C tricks to get/set the variable as efficiently as possible: use macros when compiling as a stand-alone program, but still @@ -87,9 +91,8 @@ #ifndef PYPY_NOT_MAIN_FILE - -long pypysig_occurred; -static volatile long *pypysig_occurred_v = (volatile long *)&pypysig_occurred; +struct pypysig_long_struct pypysig_occurred; +static volatile long *pypysig_occurred_v = (volatile long *)&pypysig_occurred.value; static volatile int pypysig_flags[NSIG]; void pypysig_ignore(int signum) @@ -159,7 +162,7 @@ { pypysig_flags[i] = 0; /* maybe another signal is pending: */ - pypysig_occurred |= PENDING_SIGNAL_BIT; + pypysig_occurred.value |= PENDING_SIGNAL_BIT; return i; } } Modified: pypy/branch/cpython-extension/pypy/translator/driver.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/driver.py (original) +++ pypy/branch/cpython-extension/pypy/translator/driver.py Fri Apr 23 02:41:35 2010 @@ -308,6 +308,8 @@ return res def task_annotate(self): + """ Annotate + """ # includes annotation and annotatation simplifications translator = self.translator policy = self.policy @@ -365,6 +367,8 @@ def task_rtype_lltype(self): + """ RTyping - lltype version + """ rtyper = self.translator.buildrtyper(type_system='lltype') insist = not self.config.translation.insist rtyper.specialize(dont_simplify_again=True, @@ -374,6 +378,8 @@ RTYPE = 'rtype_lltype' def task_rtype_ootype(self): + """ RTyping - ootype version + """ # Maybe type_system should simply be an option used in task_rtype insist = not self.config.translation.insist rtyper = self.translator.buildrtyper(type_system="ootype") @@ -384,6 +390,9 @@ OOTYPE = 'rtype_ootype' def task_pyjitpl_lltype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -399,6 +408,9 @@ "JIT compiler generation") def task_pyjitpl_ootype(self): + """ Generate bytecodes for JIT and flow the JIT helper functions + ootype version + """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # @@ -414,6 +426,8 @@ "JIT compiler generation") def task_backendopt_lltype(self): + """ Run all backend optimizations - lltype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -423,6 +437,8 @@ BACKENDOPT = 'backendopt_lltype' def task_backendopt_ootype(self): + """ Run all backend optimizations - ootype version + """ from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator) # @@ -453,6 +469,8 @@ raise Exception(str(e) + '\n' + i) def task_database_c(self): + """ Create a database for further backend generation + """ translator = self.translator if translator.annotator is not None: translator.frozen = True @@ -484,7 +502,9 @@ "Creating database for generating c source", earlycheck = possibly_check_for_boehm) - def task_source_c(self): # xxx messy + def task_source_c(self): + """ Create C source files from the generated database + """ translator = self.translator cbuilder = self.cbuilder database = self.database @@ -512,6 +532,8 @@ return mkexename(py.path.local(newexename)) def create_exe(self): + """ Copy the compiled executable into translator/goal + """ if self.exe_name is not None: exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() @@ -523,7 +545,10 @@ self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) - def task_compile_c(self): # xxx messy + def task_compile_c(self): + """ Compile the generated C code using either makefile or + translator/platform + """ cbuilder = self.cbuilder kwds = {} if self.standalone: Modified: pypy/branch/cpython-extension/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/geninterplevel.py (original) +++ pypy/branch/cpython-extension/pypy/translator/geninterplevel.py Fri Apr 23 02:41:35 2010 @@ -71,7 +71,7 @@ log = py.log.Producer("geninterp") py.log.setconsumer("geninterp", ansi_log) -GI_VERSION = '1.2.7' # bump this for substantial changes +GI_VERSION = '1.2.8' # bump this for substantial changes # ____________________________________________________________ try: @@ -161,11 +161,11 @@ Constant(OperationError).key: late_OperationError, Constant(Arguments).key: late_Arguments, } - u = UniqueList - self.initcode = u() # list of lines for the module's initxxx() - self.latercode = u() # list of generators generating extra lines - # for later in initxxx() -- for recursive - # objects + self.initcode = UniqueList() # list of lines for the module's initxxx() + self.latercode = UniqueList() + # list of generators generating extra lines + # for later in initxxx() -- for recursive + # objects self.namespace = NameManager() self.namespace.make_reserved_names('__doc__ __args__ space goto') self.globaldecl = [] @@ -1475,10 +1475,7 @@ """ # create something like a module if type(sourcetext) is str: - if filename is None: - code = py.code.Source(sourcetext).compile() - else: - code = NiceCompile(filename)(sourcetext) + code = py.code.Source(sourcetext).compile() else: # assume we got an already compiled source code = sourcetext Modified: pypy/branch/cpython-extension/pypy/translator/goal/nanos.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/goal/nanos.py (original) +++ pypy/branch/cpython-extension/pypy/translator/goal/nanos.py Fri Apr 23 02:41:35 2010 @@ -26,7 +26,7 @@ """ from pypy.interpreter.gateway import applevel, ObjSpace, W_Root, interp2app -import os +import os, py app_os_path = applevel(r''' from os.path import dirname, join, abspath, isfile, islink @@ -59,6 +59,6 @@ path_module_for_testing = type(os)("os.path") os_module_for_testing = type(os)("os") os_module_for_testing.path = path_module_for_testing -eval(app_os_path.code, path_module_for_testing.__dict__) -eval(app_os.code, os_module_for_testing.__dict__) +eval(py.code.Source(app_os_path.source).compile(), path_module_for_testing.__dict__) +eval(py.code.Source(app_os.source).compile(), os_module_for_testing.__dict__) Modified: pypy/branch/cpython-extension/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/cpython-extension/pypy/translator/goal/targetpypystandalone.py Fri Apr 23 02:41:35 2010 @@ -233,15 +233,17 @@ # manually imports app_main.py filename = os.path.join(this_dir, 'app_main.py') - w_dict = space.newdict() - space.exec_(open(filename).read(), w_dict, w_dict) + app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + app.hidden_applevel = False + app.can_use_geninterp = False + w_dict = app.getwdict(space) entry_point = create_entry_point(space, w_dict) return entry_point, None, PyPyAnnotatorPolicy(single_space = space) def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', - 'jitpolicy', + 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) Modified: pypy/branch/cpython-extension/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/jvm/conftest.py (original) +++ pypy/branch/cpython-extension/pypy/translator/jvm/conftest.py Fri Apr 23 02:41:35 2010 @@ -1,4 +1,12 @@ +import py, sys + +class Module(py.test.collect.Module): + def collect(self): + if sys.maxint > 2147483647: # 64bit platform + py.test.skip("jvm backend on 64bit unsupported") + return super(Module, self).collect() + def pytest_addoption(parser): group = parser.getgroup("pypy-jvm options") group.addoption('--java', action='store', dest='java', default='java', Modified: pypy/branch/cpython-extension/pypy/translator/simplify.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/simplify.py (original) +++ pypy/branch/cpython-extension/pypy/translator/simplify.py Fri Apr 23 02:41:35 2010 @@ -6,16 +6,20 @@ """ import py -from pypy.objspace.flow.model import SpaceOperation -from pypy.objspace.flow.model import Variable, Constant, Block, Link -from pypy.objspace.flow.model import c_last_exception -from pypy.objspace.flow.model import checkgraph, traverse, mkentrymap +from pypy.objspace.flow import operation +from pypy.objspace.flow.model import (SpaceOperation, Variable, Constant, Block, + Link, c_last_exception, checkgraph, + traverse, mkentrymap) +from pypy.rlib import rarithmetic +from pypy.translator import unsimplify +from pypy.translator.backendopt import ssa from pypy.rpython.lltypesystem import lloperation, lltype from pypy.rpython.ootypesystem import ootype def get_funcobj(func): """ - Return an object which is supposed to have attributes such as graph and _callable + Return an object which is supposed to have attributes such as graph and + _callable """ if hasattr(func, '_obj'): return func._obj # lltypesystem @@ -30,12 +34,9 @@ assert False def get_graph(arg, translator): - from pypy.translator.translator import graphof if isinstance(arg, Variable): return None f = arg.value - from pypy.rpython.lltypesystem import lltype - from pypy.rpython.ootypesystem import ootype if not isinstance(f, lltype._ptr) and not isinstance(f, ootype._callable): return None funcobj = get_funcobj(f) @@ -49,7 +50,7 @@ return None try: callable = funcobj._callable - return graphof(translator, callable) + return translator._graphof(callable) except (AttributeError, KeyError, AssertionError): return None @@ -109,13 +110,11 @@ ovfcheck_lshift is special because there is no preceding operation. Instead, it will be replaced by an OP_LSHIFT_OVF operation. """ - from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift - from pypy.objspace.flow.objspace import implicit_exceptions - covf = Constant(ovfcheck) - covfls = Constant(ovfcheck_lshift) + covf = Constant(rarithmetic.ovfcheck) + covfls = Constant(rarithmetic.ovfcheck_lshift) def check_syntax(opname): - exlis = implicit_exceptions.get("%s_ovf" % (opname,), []) + exlis = operation.implicit_exceptions.get("%s_ovf" % (opname,), []) if OverflowError not in exlis: raise Exception("ovfcheck in %s: Operation %s has no" " overflow variant" % (graph.name, opname)) @@ -558,8 +557,7 @@ # when for all possible incoming paths they would get twice the same # value (this is really the purpose of remove_identical_vars()). # - from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder - builder = DataFlowFamilyBuilder(graph) + builder = ssa.DataFlowFamilyBuilder(graph) variable_families = builder.get_variable_families() # vertical removal while True: if not builder.merge_identical_phi_nodes(): # horizontal removal @@ -655,8 +653,7 @@ # NB. this assumes RPythonicity: we can only iterate over something # that has a len(), and this len() cannot change as long as we are # using the iterator. - from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder - builder = DataFlowFamilyBuilder(graph) + builder = ssa.DataFlowFamilyBuilder(graph) variable_families = builder.get_variable_families() c_append = Constant('append') newlist_v = {} @@ -964,7 +961,6 @@ link.args[i] = vlist2 # - wherever the list exits the loop body, add a 'hint({fence})' - from pypy.translator.unsimplify import insert_empty_block for block in loopbody: for link in block.exits: if link.target not in loopbody: @@ -976,7 +972,7 @@ link.target in stopblocks): hints['exactlength'] = True chints = Constant(hints) - newblock = insert_empty_block(None, link) + newblock = unsimplify.insert_empty_block(None, link) index = link.args.index(vlist) vlist2 = newblock.inputargs[index] vlist3 = Variable(vlist2) Modified: pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/cpython-extension/pypy/translator/tool/cbuild.py Fri Apr 23 02:41:35 2010 @@ -1,17 +1,10 @@ -import os, sys, inspect, re, imp - import py from pypy.tool.autopath import pypydir from pypy.translator.platform import host -from pypy.tool.ansi_print import ansi_log from pypy.tool.udir import udir -log = py.log.Producer("cbuild") -py.log.setconsumer("cbuild", ansi_log) - - class ExternalCompilationInfo(object): _ATTRIBUTES = ['pre_include_bits', 'includes', 'include_dirs', Modified: pypy/branch/cpython-extension/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/branch/cpython-extension/pypy/translator/tool/test/test_cbuild.py Fri Apr 23 02:41:35 2010 @@ -1,6 +1,6 @@ -import py, sys +import py -from pypy.tool.udir import udir +from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo from subprocess import Popen, PIPE, STDOUT Modified: pypy/branch/cpython-extension/pypy/translator/translator.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/translator.py (original) +++ pypy/branch/cpython-extension/pypy/translator/translator.py Fri Apr 23 02:41:35 2010 @@ -6,9 +6,9 @@ """ import autopath, os, sys, types, copy -from pypy.objspace.flow.model import * from pypy.translator import simplify from pypy.objspace.flow.objspace import FlowObjSpace +from pypy.objspace.flow.model import FunctionGraph, checkgraph from pypy.tool.ansi_print import ansi_log from pypy.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription From fijal at codespeak.net Fri Apr 23 02:59:15 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 02:59:15 +0200 (CEST) Subject: [pypy-svn] r74004 - pypy/trunk/pypy/translator Message-ID: <20100423005915.A17EB282BAD@codespeak.net> Author: fijal Date: Fri Apr 23 02:59:14 2010 New Revision: 74004 Modified: pypy/trunk/pypy/translator/translator.py Log: fix name error (?) Modified: pypy/trunk/pypy/translator/translator.py ============================================================================== --- pypy/trunk/pypy/translator/translator.py (original) +++ pypy/trunk/pypy/translator/translator.py Fri Apr 23 02:59:14 2010 @@ -8,7 +8,7 @@ from pypy.translator import simplify from pypy.objspace.flow.objspace import FlowObjSpace -from pypy.objspace.flow.model import FunctionGraph, checkgraph +from pypy.objspace.flow.model import FunctionGraph, checkgraph, Block from pypy.tool.ansi_print import ansi_log from pypy.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription From afa at codespeak.net Fri Apr 23 10:16:17 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 10:16:17 +0200 (CEST) Subject: [pypy-svn] r74005 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100423081617.595DD282BD4@codespeak.net> Author: afa Date: Fri Apr 23 10:16:14 2010 New Revision: 74005 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: remove from stubs some recently implemented functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Fri Apr 23 10:16:14 2010 @@ -1505,34 +1505,6 @@ The delayed normalization is implemented to improve performance.""" raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_Fetch(space, ptype, pvalue, ptraceback): - """Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to NULL. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be NULL even when the type object is not. - - This function is normally only used by code that needs to handle exceptions or - by code that needs to save and restore the error indicator temporarily.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, PyObject], lltype.Void) -def PyErr_Restore(space, type, value, traceback): - """Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are NULL, the error - indicator is cleared. Do not pass a NULL type and non-NULL value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - This function is normally only used by code that needs to save and restore the - error indicator temporarily; use PyErr_Fetch() to save the current - exception state.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, ...], PyObject) def PyErr_Format(space, exception, format, ): """This function sets the error indicator and returns NULL. exception should be @@ -2834,55 +2806,6 @@ raise NotImplementedError @cpython_api([], lltype.Void) -def PyEval_InitThreads(space, ): - """ - - - - Initialize and acquire the global interpreter lock. It should be called in the - main thread before creating a second thread or engaging in any other thread - operations such as PyEval_ReleaseLock() or - PyEval_ReleaseThread(tstate). It is not needed before calling - PyEval_SaveThread() or PyEval_RestoreThread(). - - - - - - This is a no-op when called for a second time. It is safe to call this function - before calling Py_Initialize(). - - - - - - When only the main thread exists, no GIL operations are needed. This is a - common situation (most Python programs do not use threads), and the lock - operations slow the interpreter down a bit. Therefore, the lock is not - created initially. This situation is equivalent to having acquired the lock: - when there is only a single thread, all object accesses are safe. Therefore, - when this function initializes the global interpreter lock, it also acquires - it. Before the Python thread module creates a new thread, knowing - that either it has the lock or the lock hasn't been created yet, it calls - PyEval_InitThreads(). When this call returns, it is guaranteed that - the lock has been created and that the calling thread has acquired it. - - It is not safe to call this function when it is unknown which thread (if - any) currently has the global interpreter lock. - - This function is not available when thread support is disabled at compile time.""" - raise NotImplementedError - - at cpython_api([], rffi.INT_real) -def PyEval_ThreadsInitialized(space, ): - """Returns a non-zero value if PyEval_InitThreads() has been called. This - function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. This function is - not available when thread support is disabled at compile time. - """ - raise NotImplementedError - - at cpython_api([], lltype.Void) def PyEval_AcquireLock(space, ): """Acquire the global interpreter lock. The lock must have been created earlier. If this thread already has the lock, a deadlock ensues. This function is not @@ -2895,24 +2818,6 @@ This function is not available when thread support is disabled at compile time.""" raise NotImplementedError - at cpython_api([{PyThreadState*}], lltype.Void) -def PyEval_AcquireThread(space, tstate): - """Acquire the global interpreter lock and set the current thread state to - tstate, which should not be NULL. The lock must have been created earlier. - If this thread already has the lock, deadlock ensues. This function is not - available when thread support is disabled at compile time.""" - raise NotImplementedError - - at cpython_api([{PyThreadState*}], lltype.Void) -def PyEval_ReleaseThread(space, tstate): - """Reset the current thread state to NULL and release the global interpreter - lock. The lock must have been created earlier and must be held by the current - thread. The tstate argument, which must not be NULL, is only used to check - that it represents the current thread state --- if it isn't, a fatal error is - reported. This function is not available when thread support is disabled at - compile time.""" - raise NotImplementedError - @cpython_api([], {PyThreadState*}) def PyEval_SaveThread(space, ): """Release the global interpreter lock (if it has been created and thread @@ -2958,26 +2863,6 @@ PyInterpreterState_Clear().""" raise NotImplementedError - at cpython_api([{PyInterpreterState*}], {PyThreadState*}) -def PyThreadState_New(space, interp): - """Create a new thread state object belonging to the given interpreter object. - The global interpreter lock need not be held, but may be held if it is - necessary to serialize calls to this function.""" - raise NotImplementedError - - at cpython_api([{PyThreadState*}], lltype.Void) -def PyThreadState_Clear(space, tstate): - """Reset all information in a thread state object. The global interpreter lock - must be held.""" - raise NotImplementedError - - at cpython_api([{PyThreadState*}], lltype.Void) -def PyThreadState_Delete(space, tstate): - """Destroy a thread state object. The global interpreter lock need not be held. - The thread state must have been reset with a previous call to - PyThreadState_Clear().""" - raise NotImplementedError - @cpython_api([], {PyThreadState*}) def PyThreadState_Get(space, ): """Return the current thread state. The global interpreter lock must be held. @@ -2985,12 +2870,6 @@ the caller needn't check for NULL).""" raise NotImplementedError - at cpython_api([{PyThreadState*}], {PyThreadState*}) -def PyThreadState_Swap(space, tstate): - """Swap the current thread state with the thread state given by the argument - tstate, which may be NULL. The global interpreter lock must be held.""" - raise NotImplementedError - @cpython_api([], PyObject, borrowed=True) def PyThreadState_GetDict(space, ): """Return a dictionary in which extensions can store thread-specific state @@ -3393,12 +3272,6 @@ """ raise NotImplementedError - at cpython_api([{double}], PyObject) -def PyLong_FromDouble(space, v): - """Return a new PyLongObject object from the integer part of v, or - NULL on failure.""" - raise NotImplementedError - @cpython_api([{Py_UNICODE*}, Py_ssize_t, rffi.INT_real], PyObject) def PyLong_FromUnicode(space, u, length, base): """Convert a sequence of Unicode digits to a Python long integer value. The first @@ -3423,17 +3296,6 @@ If the integer is larger than LONG_MAX, a positive long integer is returned.""" raise NotImplementedError - at cpython_api([PyObject], lltype.Signed) -def PyLong_AsLong(space, pylong): - """ - - - - Return a C long representation of the contents of pylong. If - pylong is greater than LONG_MAX, an OverflowError is raised - and -1 will be returned.""" - raise NotImplementedError - @cpython_api([PyObject, {int*}], lltype.Signed) def PyLong_AsLongAndOverflow(space, pylong, overflow): """Return a C long representation of the contents of @@ -3891,16 +3753,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyNumber_Long(space, o): - """ - - - - Returns the o converted to a long integer object on success, or NULL on - failure. This is the equivalent of the Python expression long(o).""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyNumber_Index(space, o): """Returns the o converted to a Python int or long on success or NULL with a TypeError exception raised on failure. @@ -4006,16 +3858,6 @@ o.attr_name = v.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real) -def PyObject_GenericSetAttr(space, o, name, value): - """Generic attribute setter function that is meant to be put into a type - object's tp_setattro slot. It looks for a data descriptor in the - dictionary of classes in the object's MRO, and if found it takes preference - over setting the attribute in the instance dictionary. Otherwise, the - attribute is set in the object's __dict__ (if present). Otherwise, - an AttributeError is raised and -1 is returned.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], rffi.INT_real) def PyObject_DelAttr(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. @@ -4075,30 +3917,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyObject_Repr(space, o): - """ - - - - Compute a string representation of object o. Returns the string - representation on success, NULL on failure. This is the equivalent of the - Python expression repr(o). Called by the repr() built-in function and - by reverse quotes.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyObject_Str(space, o): - """ - - - - Compute a string representation of object o. Returns the string - representation on success, NULL on failure. This is the equivalent of the - Python expression str(o). Called by the str() built-in function and - by the print statement.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyObject_Bytes(space, o): """ @@ -4913,14 +4731,6 @@ This function is not available in 3.x and does not have a PyBytes alias.""" raise NotImplementedError - at cpython_api([{PyMethodDef}, PyObject, rffi.CCHARP], PyObject) -def Py_FindMethod(space, table[], ob, name): - """Return a bound method object for an extension type implemented in C. This - can be useful in the implementation of a tp_getattro or - tp_getattr handler that does not use the - PyObject_GenericGetAttr() function.""" - raise NotImplementedError - @cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) def Py_FdIsInteractive(space, fp, filename): """Return true (nonzero) if the standard I/O file fp with name filename is @@ -5255,14 +5065,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyUnicode_GetSize(space, unicode): - """Return the length of the Unicode object. - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) def PyUnicode_FromEncodedObject(space, obj, encoding, errors): """Coerce an encoded object obj to an Unicode object and return a reference with @@ -5627,15 +5429,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) -def PyUnicode_DecodeMBCS(space, s, size, errors): - """Create a Unicode object by decoding size bytes of the MBCS encoded string s. - Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.INT_real, rffi.CCHARP, {int*}], PyObject) def PyUnicode_DecodeMBCSStateful(space, s, size, errors, consumed): """If consumed is NULL, behave like PyUnicode_DecodeMBCS(). If @@ -5645,15 +5438,6 @@ """ raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP], PyObject) -def PyUnicode_EncodeMBCS(space, s, size, errors): - """Encode the Py_UNICODE buffer of the given size using MBCS and return a - Python string object. Return NULL if an exception was raised by the codec. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyUnicode_AsMBCSString(space, unicode): """Encode a Unicode object using MBCS and return the result as Python string From afa at codespeak.net Fri Apr 23 10:55:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 10:55:19 +0200 (CEST) Subject: [pypy-svn] r74006 - pypy/branch/cpython-extension/pypy/translator Message-ID: <20100423085519.6BEA0282BD4@codespeak.net> Author: afa Date: Fri Apr 23 10:55:17 2010 New Revision: 74006 Modified: pypy/branch/cpython-extension/pypy/translator/translator.py Log: merge r74004 from trunk Modified: pypy/branch/cpython-extension/pypy/translator/translator.py ============================================================================== --- pypy/branch/cpython-extension/pypy/translator/translator.py (original) +++ pypy/branch/cpython-extension/pypy/translator/translator.py Fri Apr 23 10:55:17 2010 @@ -8,7 +8,7 @@ from pypy.translator import simplify from pypy.objspace.flow.objspace import FlowObjSpace -from pypy.objspace.flow.model import FunctionGraph, checkgraph +from pypy.objspace.flow.model import FunctionGraph, checkgraph, Block from pypy.tool.ansi_print import ansi_log from pypy.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription From afa at codespeak.net Fri Apr 23 13:26:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:26:05 +0200 (CEST) Subject: [pypy-svn] r74008 - pypy/trunk/pypy/interpreter Message-ID: <20100423112605.19BFC282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:26:03 2010 New Revision: 74008 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py Log: Fix translation: inside ObjSpace, the space is named 'self'... Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Apr 23 13:26:03 2010 @@ -717,8 +717,8 @@ raise break # done if expected_length != -1 and len(items) == expected_length: - raise OperationError(space.w_ValueError, - space.wrap("too many values to unpack")) + raise OperationError(self.w_ValueError, + self.wrap("too many values to unpack")) items.append(w_item) if expected_length != -1 and len(items) < expected_length: i = len(items) @@ -726,8 +726,8 @@ plural = "" else: plural = "s" - raise OperationError(space.w_ValueError, - space.wrap("need more than %d value%s to unpack" % + raise OperationError(self.w_ValueError, + self.wrap("need more than %d value%s to unpack" % (i, plural))) return items From afa at codespeak.net Fri Apr 23 13:28:26 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:28:26 +0200 (CEST) Subject: [pypy-svn] r74009 - pypy/trunk/pypy/interpreter Message-ID: <20100423112826.63A5B282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:28:25 2010 New Revision: 74009 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py Log: More space->self renaming Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Apr 23 13:28:25 2010 @@ -670,7 +670,7 @@ w_s = self.interned_strings[s] = self.wrap(s) return w_s - def interpclass_w(space, w_obj): + def interpclass_w(self, w_obj): """ If w_obj is a wrapped internal interpreter class instance unwrap to it, otherwise return None. (Can be overridden in specific spaces; you @@ -728,7 +728,7 @@ plural = "s" raise OperationError(self.w_ValueError, self.wrap("need more than %d value%s to unpack" % - (i, plural))) + (i, plural))) return items def fixedview(self, w_iterable, expected_length=-1): @@ -1103,16 +1103,16 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(space, w_obj): + def path_w(self, w_obj): """ Like str_w, but if the object is unicode, encode it using filesystemencoding """ - filesystemencoding = space.sys.filesystemencoding + filesystemencoding = self.sys.filesystemencoding if (filesystemencoding and - space.is_true(space.isinstance(w_obj, space.w_unicode))): - w_obj = space.call_method(w_obj, "encode", - space.wrap(filesystemencoding)) - return space.str_w(w_obj) + self.is_true(self.isinstance(w_obj, self.w_unicode))): + w_obj = self.call_method(w_obj, "encode", + self.wrap(filesystemencoding)) + return self.str_w(w_obj) def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. From afa at codespeak.net Fri Apr 23 13:30:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:30:45 +0200 (CEST) Subject: [pypy-svn] r74010 - pypy/branch/cpython-extension/pypy/interpreter Message-ID: <20100423113045.A4351282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:30:43 2010 New Revision: 74010 Modified: pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py Log: Merge r74007 and r74008 from trunk Modified: pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/cpython-extension/pypy/interpreter/baseobjspace.py Fri Apr 23 13:30:43 2010 @@ -670,7 +670,7 @@ w_s = self.interned_strings[s] = self.wrap(s) return w_s - def interpclass_w(space, w_obj): + def interpclass_w(self, w_obj): """ If w_obj is a wrapped internal interpreter class instance unwrap to it, otherwise return None. (Can be overridden in specific spaces; you @@ -717,8 +717,8 @@ raise break # done if expected_length != -1 and len(items) == expected_length: - raise OperationError(space.w_ValueError, - space.wrap("too many values to unpack")) + raise OperationError(self.w_ValueError, + self.wrap("too many values to unpack")) items.append(w_item) if expected_length != -1 and len(items) < expected_length: i = len(items) @@ -726,9 +726,9 @@ plural = "" else: plural = "s" - raise OperationError(space.w_ValueError, - space.wrap("need more than %d value%s to unpack" % - (i, plural))) + raise OperationError(self.w_ValueError, + self.wrap("need more than %d value%s to unpack" % + (i, plural))) return items def fixedview(self, w_iterable, expected_length=-1): @@ -1103,16 +1103,16 @@ self.wrap('argument must be a unicode')) return self.unicode_w(w_obj) - def path_w(space, w_obj): + def path_w(self, w_obj): """ Like str_w, but if the object is unicode, encode it using filesystemencoding """ - filesystemencoding = space.sys.filesystemencoding + filesystemencoding = self.sys.filesystemencoding if (filesystemencoding and - space.is_true(space.isinstance(w_obj, space.w_unicode))): - w_obj = space.call_method(w_obj, "encode", - space.wrap(filesystemencoding)) - return space.str_w(w_obj) + self.is_true(self.isinstance(w_obj, self.w_unicode))): + w_obj = self.call_method(w_obj, "encode", + self.wrap(filesystemencoding)) + return self.str_w(w_obj) def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. From afa at codespeak.net Fri Apr 23 13:32:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:32:08 +0200 (CEST) Subject: [pypy-svn] r74011 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100423113208.369DE282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:32:06 2010 New Revision: 74011 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Log: PySequence_Concat Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Fri Apr 23 13:32:06 2010 @@ -87,3 +87,9 @@ otherwise a tuple will be constructed with the appropriate contents. This is equivalent to the Python expression tuple(o).""" return space.call_function(space.w_tuple, w_obj) + + at cpython_api([PyObject, PyObject], PyObject) +def PySequence_Concat(space, w_o1, w_o2): + """Return the concatenation of o1 and o2 on success, and NULL on failure. + This is the equivalent of the Python expression o1 + o2.""" + return space.add(w_o1, w_o2) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Fri Apr 23 13:32:06 2010 @@ -3,7 +3,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext import sequence -class TestIterator(BaseApiTest): +class TestSequence(BaseApiTest): def test_sequence(self, space, api): w_t = space.wrap((1, 2, 3, 4)) assert api.PySequence_Fast(w_t, "message") is w_t @@ -22,6 +22,11 @@ assert space.type(w_seq) is space.w_tuple assert sorted(space.unwrap(w_seq)) == [1, 2, 3, 4] + def test_concat(self, space, api): + w_t1 = space.wrap(range(4)) + w_t2 = space.wrap(range(4, 8)) + assert space.unwrap(api.PySequence_Concat(w_t1, w_t2)) == range(8) + def test_exception(self, space, api): message = rffi.str2charp("message") assert not api.PySequence_Fast(space.wrap(3), message) From afa at codespeak.net Fri Apr 23 13:33:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:33:19 +0200 (CEST) Subject: [pypy-svn] r74012 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100423113319.C897E282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:33:18 2010 New Revision: 74012 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Log: Add T_BYTE in structmember Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/structmember.h Fri Apr 23 13:33:18 2010 @@ -27,6 +27,7 @@ #define T_STRING 5 #define T_OBJECT 6 #define T_CHAR 7 /* 1-character string */ +#define T_BYTE 8 /* 8-bit signed int */ #define T_USHORT 10 #define T_UINT 11 #define T_ULONG 12 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmember.py Fri Apr 23 13:33:18 2010 @@ -34,6 +34,9 @@ elif member_type == structmemberdefs.T_ULONG: result = rffi.cast(rffi.ULONGP, addr) w_result = space.wrap(result[0]) + elif member_type == structmemberdefs.T_BYTE: + result = rffi.cast(rffi.CCHARP, addr) + w_result = space.wrap(result[0]) elif member_type == structmemberdefs.T_STRING: result = rffi.cast(rffi.CCHARPP, addr) if result[0]: @@ -107,6 +110,10 @@ w_long_value = PyInt_AsUnsignedLong(space, w_value) array = rffi.cast(rffi.ULONGP, addr) array[0] = rffi.cast(rffi.ULONG, w_long_value) + elif member_type == structmemberdefs.T_BYTE: + w_long_value = PyInt_AsLong(space, w_value) + array = rffi.cast(rffi.CCHARP, addr) + array[0] = rffi.cast(rffi.CHAR, w_long_value) elif member_type == structmemberdefs.T_CHAR: str_value = space.str_w(w_value) if len(str_value) != 1: From afa at codespeak.net Fri Apr 23 13:34:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:34:00 +0200 (CEST) Subject: [pypy-svn] r74013 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100423113400.2D134282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:33:58 2010 New Revision: 74013 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Log: This belongs to the previous commit Modified: pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/structmemberdefs.py Fri Apr 23 13:33:58 2010 @@ -4,6 +4,7 @@ T_STRING = 5 T_OBJECT = 6 T_CHAR = 7 +T_BYTE = 8 T_USHORT = 10 T_UINT = 11 T_ULONG = 12 From afa at codespeak.net Fri Apr 23 13:35:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:35:39 +0200 (CEST) Subject: [pypy-svn] r74014 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100423113539.19ED8282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:35:37 2010 New Revision: 74014 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Log: - Add #include on Windows like CPython does - move declaration of PyOS_snprintf to a place enclosed by "extern C" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Fri Apr 23 13:35:37 2010 @@ -13,6 +13,7 @@ #else # define MS_WIN32 1 # include +# include # define Py_DEPRECATED(VERSION_UNUSED) # ifdef Py_BUILD_CORE # define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE @@ -79,7 +80,6 @@ #include -int PyOS_snprintf(char *str, size_t size, const char *format, ...); #include "patchlevel.h" #include "object.h" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/stringobject.h Fri Apr 23 13:35:37 2010 @@ -7,6 +7,8 @@ extern "C" { #endif +int PyOS_snprintf(char *str, size_t size, const char *format, ...); + #define PyString_GET_SIZE(op) PyString_Size(op) #define PyString_AS_STRING(op) PyString_AsString(op) From afa at codespeak.net Fri Apr 23 13:39:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:39:54 +0200 (CEST) Subject: [pypy-svn] r74015 - pypy/trunk/pypy/module/_locale Message-ID: <20100423113954.C2D09282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:39:53 2010 New Revision: 74015 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py Log: cConfig is now in rlib.rlocale Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Fri Apr 23 13:39:53 2010 @@ -319,11 +319,11 @@ buf_country = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') try: - if (GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_SISO639LANGNAME, + if (GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_SISO639LANGNAME, buf_lang, BUFSIZE) and - GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_SISO3166CTRYNAME, + GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_SISO3166CTRYNAME, buf_country, BUFSIZE)): lang = rffi.charp2str(buf_lang) country = rffi.charp2str(buf_country) @@ -333,8 +333,8 @@ # If we end up here, this windows version didn't know about # ISO639/ISO3166 names (it's probably Windows 95). Return the # Windows language identifier instead (a hexadecimal number) - elif GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_IDEFAULTLANGUAGE, + elif GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_IDEFAULTLANGUAGE, buf_lang, BUFSIZE): lang = rffi.charp2str(buf_lang) return space.newtuple([space.wrap("0x%s" % lang), From afa at codespeak.net Fri Apr 23 13:55:01 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 13:55:01 +0200 (CEST) Subject: [pypy-svn] r74016 - pypy/branch/cpython-extension/pypy/module/_locale Message-ID: <20100423115501.C3313282BD4@codespeak.net> Author: afa Date: Fri Apr 23 13:55:00 2010 New Revision: 74016 Modified: pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py Log: Merge r74015 from trunk Modified: pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/cpython-extension/pypy/module/_locale/interp_locale.py Fri Apr 23 13:55:00 2010 @@ -319,11 +319,11 @@ buf_country = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') try: - if (GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_SISO639LANGNAME, + if (GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_SISO639LANGNAME, buf_lang, BUFSIZE) and - GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_SISO3166CTRYNAME, + GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_SISO3166CTRYNAME, buf_country, BUFSIZE)): lang = rffi.charp2str(buf_lang) country = rffi.charp2str(buf_country) @@ -333,8 +333,8 @@ # If we end up here, this windows version didn't know about # ISO639/ISO3166 names (it's probably Windows 95). Return the # Windows language identifier instead (a hexadecimal number) - elif GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, - cConfig.LOCALE_IDEFAULTLANGUAGE, + elif GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, + rlocale.cConfig.LOCALE_IDEFAULTLANGUAGE, buf_lang, BUFSIZE): lang = rffi.charp2str(buf_lang) return space.newtuple([space.wrap("0x%s" % lang), From afa at codespeak.net Fri Apr 23 14:11:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 14:11:59 +0200 (CEST) Subject: [pypy-svn] r74017 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100423121159.5EBB4282BD4@codespeak.net> Author: afa Date: Fri Apr 23 14:11:57 2010 New Revision: 74017 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Log: PyMapping_SetItemString, PyMapping_GetItemString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Fri Apr 23 14:11:57 2010 @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem import lltype, rffi -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.cpyext.pyobject import PyObject @@ -22,3 +22,18 @@ the Python expression o.items().""" return space.call_method(w_obj, "items") + at cpython_api([PyObject, CONST_STRING], PyObject) +def PyMapping_GetItemString(space, w_obj, key): + """Return element of o corresponding to the object key or NULL on failure. + This is the equivalent of the Python expression o[key].""" + w_key = space.wrap(rffi.charp2str(key)) + return space.getitem(w_obj, w_key) + + at cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1) +def PyMapping_SetItemString(space, w_obj, key, w_value): + """Map the object key to the value v in object o. Returns -1 on failure. + This is the equivalent of the Python statement o[key] = v.""" + w_key = space.wrap(rffi.charp2str(key)) + space.setitem(w_obj, w_key, w_value) + return 0 + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Fri Apr 23 14:11:57 2010 @@ -1,5 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest - +from pypy.rpython.lltypesystem import lltype, rffi class TestMapping(BaseApiTest): def test_check(self, space, api): @@ -15,3 +15,11 @@ w_d = space.newdict() space.setitem(w_d, space.wrap("a"), space.wrap("b")) assert space.eq_w(api.PyMapping_Items(w_d), space.wrap([("a", "b")])) + + def test_setitemstring(self, space, api): + w_d = space.newdict() + key = rffi.str2charp("key") + api.PyMapping_SetItemString(w_d, key, space.wrap(42)) + assert 42 == space.unwrap( + api.PyMapping_GetItemString(w_d, key, space.wrap(42))) + rffi.free_charp(key) From afa at codespeak.net Fri Apr 23 14:55:39 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 14:55:39 +0200 (CEST) Subject: [pypy-svn] r74018 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100423125539.EACFA282BD4@codespeak.net> Author: afa Date: Fri Apr 23 14:55:38 2010 New Revision: 74018 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Log: PySys_SetObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sysmodule.py Fri Apr 23 14:55:38 2010 @@ -1,14 +1,24 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api +from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api, CONST_STRING from pypy.module.cpyext.pyobject import PyObject, register_container - at cpython_api([rffi.CCHARP], PyObject, borrowed=True, error=CANNOT_FAIL) + at cpython_api([CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) def PySys_GetObject(space, name): """Return the object name from the sys module or NULL if it does not exist, without setting an exception.""" - w_name = rffi.charp2str(name) + name = rffi.charp2str(name) w_dict = space.sys.getdict() - w_obj = space.finditem_str(w_dict, w_name) + w_obj = space.finditem_str(w_dict, name) register_container(space, w_dict) return w_obj + + at cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) +def PySys_SetObject(space, name, w_obj): + """Set name in the sys module to v unless v is NULL, in which + case name is deleted from the sys module. Returns 0 on success, -1 + on error.""" + name = rffi.charp2str(name) + w_dict = space.sys.getdict() + space.setitem_str(w_dict, name, w_obj) + return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sysmodule.py Fri Apr 23 14:55:38 2010 @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.rpython.lltypesystem import rffi class AppTestSysModule(AppTestCpythonExtensionBase): @@ -13,3 +14,9 @@ assert module.get("excepthook") assert not module.get("spam_spam_spam") +class TestSysModule(BaseApiTest): + def test_sysmodule(self, space, api): + buf = rffi.str2charp("last_tb") + api.PySys_SetObject(buf, space.wrap(1)) + rffi.free_charp(buf) + assert space.unwrap(space.sys.get("last_tb")) == 1 From arigo at codespeak.net Fri Apr 23 15:19:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Apr 2010 15:19:48 +0200 (CEST) Subject: [pypy-svn] r74019 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100423131948.517E8282BD4@codespeak.net> Author: arigo Date: Fri Apr 23 15:19:46 2010 New Revision: 74019 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Put the functions from jitter.py in a class, in order to give them access to the cpu. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri Apr 23 15:19:46 2010 @@ -8,8 +8,8 @@ class CodeWriter(object): - def __init__(self, rtyper=None): - self.rtyper = rtyper + def __init__(self, cpu=None): + self.cpu = cpu self.assembler = Assembler() def transform_func_to_jitcode(self, func, values, type_system='lltype'): @@ -19,7 +19,7 @@ return self.transform_graph_to_jitcode(graph) def transform_graph_to_jitcode(self, graph, verbose=False): - transform_graph(graph) + transform_graph(graph, self.cpu) regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Fri Apr 23 15:19:46 2010 @@ -4,98 +4,110 @@ from pypy.jit.codewriter.flatten import ListOfKind -def transform_graph(graph): +def transform_graph(graph, cpu=None): """Transform a control flow graph to make it suitable for being flattened in a JitCode. """ - for block in graph.iterblocks(): - rename = {} - newoperations = [] - for op in block.operations: - if is_noop_operation(op): - rename[op.result] = rename.get(op.args[0], op.args[0]) - else: - for i, v in enumerate(op.args): - if v in rename: - op = SpaceOperation(op.opname, op.args[:], op.result) - op.args[i] = rename[v] - newoperations.append(rewrite_operation(op)) - block.operations = newoperations - optimize_goto_if_not(block) - - -def optimize_goto_if_not(block): - """Replace code like 'v = int_gt(x,y); exitswitch = v' - with just 'exitswitch = ('int_gt',x,y)'.""" - if len(block.exits) != 2: - return False - v = block.exitswitch - if v.concretetype != lltype.Bool: + t = Transformer(cpu) + t.transform(graph) + + +class Transformer(object): + + def __init__(self, cpu=None): + self.cpu = cpu + + def transform(self, graph): + for block in graph.iterblocks(): + rename = {} + newoperations = [] + for op in block.operations: + if self.is_noop_operation(op): + rename[op.result] = rename.get(op.args[0], op.args[0]) + else: + for i, v in enumerate(op.args): + if v in rename: + op = SpaceOperation(op.opname, op.args[:], + op.result) + op.args[i] = rename[v] + newoperations.append(self.rewrite_operation(op)) + block.operations = newoperations + self.optimize_goto_if_not(block) + + # ---------- + + def optimize_goto_if_not(self, block): + """Replace code like 'v = int_gt(x,y); exitswitch = v' + with just 'exitswitch = ('int_gt',x,y)'.""" + if len(block.exits) != 2: + return False + v = block.exitswitch + if v.concretetype != lltype.Bool: + return False + for link in block.exits: + if v in link.args: + return False # variable escapes to next block + for op in block.operations[::-1]: + if v in op.args: + return False # variable is also used in cur block + if v is op.result: + if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', + 'int_gt', 'int_ge'): + return False # not a supported operation + # ok! optimize this case + block.operations.remove(op) + block.exitswitch = (op.opname,) + tuple(op.args) + return True return False - for link in block.exits: - if v in link.args: - return False # variable escapes to next block - for op in block.operations[::-1]: - if v in op.args: - return False # variable is also used in cur block - if v is op.result: - if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', - 'int_gt', 'int_ge'): - return False # not a supported operation - # ok! optimize this case - block.operations.remove(op) - block.exitswitch = (op.opname,) + tuple(op.args) - return True - return False -# ____________________________________________________________ + # ---------- -def is_noop_operation(op): - return op.opname in _noop_operations + def is_noop_operation(self, op): + return op.opname in self._noop_operations -_noop_operations = {'same_as': True, - 'cast_int_to_char': True, - 'cast_char_to_int': True} - -def rewrite_operation(op): - try: - return _rewrite_ops[op.opname](op) - except KeyError: - return op - -def rewrite_op_direct_call(op): - """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' - into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. - The name is one of 'residual_call_{r,ir,irf}_{i,r,f,v}'.""" - args_i = [] - args_r = [] - args_f = [] - for v in op.args[1:]: - add_in_correct_list(v, args_i, args_r, args_f) - if args_f: kinds = 'irf' - elif args_i: kinds = 'ir' - else: kinds = 'r' - sublists = [] - if 'i' in kinds: sublists.append(ListOfKind('int', args_i)) - if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) - if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) - reskind = getkind(op.result.concretetype)[0] - return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), - [op.args[0]] + sublists, - op.result) - -def add_in_correct_list(v, lst_i, lst_r, lst_f): - kind = getkind(v.concretetype) - if kind == 'void': return - elif kind == 'int': lst = lst_i - elif kind == 'ref': lst = lst_r - elif kind == 'float': lst = lst_f - else: raise AssertionError(kind) - lst.append(v) + _noop_operations = {'same_as': True, + 'cast_int_to_char': True, + 'cast_char_to_int': True} + + def rewrite_operation(self, op): + try: + return _rewrite_ops[op.opname](self, op) + except KeyError: + return op + + def rewrite_op_direct_call(self, op): + """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' + into e.g. 'i0 = residual_call_ir_i(fn, [i1, i2], [ref1, ref2])'. + The name is one of 'residual_call_{r,ir,irf}_{i,r,f,v}'.""" + args_i = [] + args_r = [] + args_f = [] + for v in op.args[1:]: + self.add_in_correct_list(v, args_i, args_r, args_f) + if args_f: kinds = 'irf' + elif args_i: kinds = 'ir' + else: kinds = 'r' + sublists = [] + if 'i' in kinds: sublists.append(ListOfKind('int', args_i)) + if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) + if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) + reskind = getkind(op.result.concretetype)[0] + return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), + [op.args[0]] + sublists, + op.result) + + def add_in_correct_list(self, v, lst_i, lst_r, lst_f): + kind = getkind(v.concretetype) + if kind == 'void': return + elif kind == 'int': lst = lst_i + elif kind == 'ref': lst = lst_r + elif kind == 'float': lst = lst_f + else: raise AssertionError(kind) + lst.append(v) # ____________________________________________________________ _rewrite_ops = {} -for _name, _func in globals().items(): +for _name in dir(Transformer): if _name.startswith('rewrite_op_'): - _rewrite_ops[_name[len('rewrite_op_'):]] = _func + _rewrite_ops[_name[len('rewrite_op_'):]] = getattr(Transformer, _name) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Fri Apr 23 15:19:46 2010 @@ -1,7 +1,7 @@ import random from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import SpaceOperation, Variable, Constant -from pypy.jit.codewriter import jitter +from pypy.jit.codewriter.jitter import Transformer from pypy.jit.metainterp.history import getkind from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.translator.unsimplify import varoftype @@ -19,7 +19,7 @@ block.operations = [sp1, SpaceOperation('int_gt', [v1, v2], v3), sp2] block.exitswitch = v3 block.exits = exits = [FakeLink(), FakeLink()] - res = jitter.optimize_goto_if_not(block) + res = Transformer().optimize_goto_if_not(block) assert res == True assert block.operations == [sp1, sp2] assert block.exitswitch == ('int_gt', v1, v2) @@ -30,7 +30,7 @@ block = Block([v1]) block.exitswitch = v1 block.exits = [FakeLink(), FakeLink()] - assert not jitter.optimize_goto_if_not(block) + assert not Transformer().optimize_goto_if_not(block) def test_optimize_goto_if_not__exit(): v1 = Variable() @@ -41,7 +41,7 @@ block.exitswitch = v3 block.exits = [FakeLink(), FakeLink()] block.exits[1].args = [v3] - assert not jitter.optimize_goto_if_not(block) + assert not Transformer().optimize_goto_if_not(block) def test_optimize_goto_if_not__unknownop(): v3 = Variable(); v3.concretetype = lltype.Bool @@ -49,7 +49,7 @@ block.operations = [SpaceOperation('foobar', [], v3)] block.exitswitch = v3 block.exits = [FakeLink(), FakeLink()] - assert not jitter.optimize_goto_if_not(block) + assert not Transformer().optimize_goto_if_not(block) def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, @@ -78,7 +78,7 @@ def residual_call_test(argtypes, restype, expectedkind): op = get_direct_call_op(argtypes, restype) - op1 = jitter.rewrite_operation(op) + op1 = Transformer().rewrite_operation(op) reskind = getkind(restype)[0] assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result From afa at codespeak.net Fri Apr 23 15:23:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 23 Apr 2010 15:23:44 +0200 (CEST) Subject: [pypy-svn] r74020 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100423132344.43EA4282BD4@codespeak.net> Author: afa Date: Fri Apr 23 15:23:42 2010 New Revision: 74020 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: PyList_SetSlice Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Fri Apr 23 15:23:42 2010 @@ -102,3 +102,17 @@ space.call_method(w_list, "reverse") return 0 + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) +def PyList_SetSlice(space, w_list, low, high, w_sequence): + """Set the slice of list between low and high to the contents of + itemlist. Analogous to list[low:high] = itemlist. The itemlist may + be NULL, indicating the assignment of an empty list (slice deletion). + Return 0 on success, -1 on failure. Negative indices, as when + slicing from Python, are not supported.""" + w_start = space.wrap(low) + w_stop = space.wrap(high) + if w_sequence: + space.setslice(w_list, w_start, w_stop, w_sequence) + else: + space.delslice(w_list, w_start, w_stop) + return 0 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Fri Apr 23 15:23:42 2010 @@ -74,6 +74,16 @@ Py_RETURN_NONE; """ ), + ("setslice", "METH_VARARGS", + """ + PyObject *l = PyTuple_GetItem(args, 0); + PyObject *seq = PyTuple_GetItem(args, 1); + if (seq == Py_None) seq = NULL; + if (PyList_SetSlice(l, 1, 4, seq) < 0) + return NULL; + Py_RETURN_NONE; + """ + ), ]) l = module.newlist() assert l == [3, -5, 1000] @@ -95,3 +105,11 @@ assert len(l) == 1 assert l[0] == 14 + l = range(6) + module.setslice(l, ['a']) + assert l == [0, 'a', 4, 5] + + l = range(6) + module.setslice(l, None) + assert l == [0, 4, 5] + From arigo at codespeak.net Fri Apr 23 16:45:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Apr 2010 16:45:13 +0200 (CEST) Subject: [pypy-svn] r74021 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100423144513.CCB44282BD4@codespeak.net> Author: arigo Date: Fri Apr 23 16:45:12 2010 New Revision: 74021 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Stick the calldescr in a residual_call operation. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Fri Apr 23 16:45:12 2010 @@ -92,8 +92,11 @@ if 'r' in kinds: sublists.append(ListOfKind('ref', args_r)) if 'f' in kinds: sublists.append(ListOfKind('float', args_f)) reskind = getkind(op.result.concretetype)[0] + FUNC = op.args[0].concretetype.TO + NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) + calldescr = self.cpu.calldescrof(FUNC, NONVOIDARGS, FUNC.RESULT) return SpaceOperation('residual_call_%s_%s' % (kinds, reskind), - [op.args[0]] + sublists, + [op.args[0], calldescr] + sublists, op.result) def add_in_correct_list(self, v, lst_i, lst_r, lst_f): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Fri Apr 23 16:45:12 2010 @@ -6,6 +6,10 @@ from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.translator.unsimplify import varoftype +class FakeCPU: + def calldescrof(self, FUNC, ARGS, RESULT): + return ('calldescr', FUNC, ARGS, RESULT) + class FakeLink: args = [] @@ -54,18 +58,20 @@ def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: + for with_void in [False, True]: for with_i in [False, True]: - for with_r in [False, True]: - for with_f in [False, True]: - ARGS = [] - if with_i: ARGS += [lltype.Signed, lltype.Char] - if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] - if with_f: ARGS += [lltype.Float, lltype.Float] - random.shuffle(ARGS) - if with_f: expectedkind = 'irf' # all kinds - elif with_i: expectedkind = 'ir' # integers and references - else: expectedkind = 'r' # only references - yield residual_call_test, ARGS, RESTYPE, expectedkind + for with_r in [False, True]: + for with_f in [False, True]: + ARGS = [] + if with_void: ARGS += [lltype.Void, lltype.Void] + if with_i: ARGS += [lltype.Signed, lltype.Char] + if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)] + if with_f: ARGS += [lltype.Float, lltype.Float] + random.shuffle(ARGS) + if with_f: expectedkind = 'irf' # all kinds + elif with_i: expectedkind = 'ir' # integers and references + else: expectedkind = 'r' # only references + yield residual_call_test, ARGS, RESTYPE, expectedkind def get_direct_call_op(argtypes, restype): FUNC = lltype.FuncType(argtypes, restype) @@ -78,13 +84,19 @@ def residual_call_test(argtypes, restype, expectedkind): op = get_direct_call_op(argtypes, restype) - op1 = Transformer().rewrite_operation(op) + op1 = Transformer(FakeCPU()).rewrite_operation(op) reskind = getkind(restype)[0] assert op1.opname == 'residual_call_%s_%s' % (expectedkind, reskind) assert op1.result == op.result assert op1.args[0] == op.args[0] - assert len(op1.args) == 1 + len(expectedkind) - for sublist, kind1 in zip(op1.args[1:], expectedkind): + FUNC = op.args[0].concretetype.TO + NONVOIDARGS = tuple([ARG for ARG in FUNC.ARGS if ARG != lltype.Void]) + assert op1.args[1] == ('calldescr', FUNC, NONVOIDARGS, FUNC.RESULT) + assert len(op1.args) == 2 + len(expectedkind) + for sublist, kind1 in zip(op1.args[2:], expectedkind): assert sublist.kind.startswith(kind1) assert list(sublist) == [v for v in op.args[1:] if getkind(v.concretetype) == sublist.kind] + for v in op.args[1:]: + kind = getkind(v.concretetype) + assert kind == 'void' or kind[0] in expectedkind From arigo at codespeak.net Fri Apr 23 16:50:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Apr 2010 16:50:17 +0200 (CEST) Subject: [pypy-svn] r74022 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100423145017.F3BEA282BD4@codespeak.net> Author: arigo Date: Fri Apr 23 16:50:16 2010 New Revision: 74022 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Log: Flattening of AbstractDescr. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Fri Apr 23 16:50:16 2010 @@ -1,5 +1,5 @@ from pypy.objspace.flow.model import Variable, Constant -from pypy.jit.metainterp.history import getkind +from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.rpython.lltypesystem import lltype @@ -189,6 +189,8 @@ elif isinstance(v, ListOfKind): lst = [self.getcolor(x) for x in v] v = ListOfKind(v.kind, lst) + elif isinstance(v, AbstractDescr): + pass else: raise NotImplementedError(type(v)) args.append(v) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Fri Apr 23 16:50:16 2010 @@ -2,6 +2,7 @@ from pypy.objspace.flow.model import Constant from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.metainterp.history import AbstractDescr def format_assembler(ssarepr, dump=True): @@ -18,6 +19,8 @@ return getlabelname(x) elif isinstance(x, ListOfKind): return '%s[%s]' % (x.kind[0], ', '.join(map(repr, x))) + elif isinstance(x, AbstractDescr): + return '%r' % (x,) else: return '' % (x,) # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri Apr 23 16:50:16 2010 @@ -4,6 +4,7 @@ from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitter import transform_graph +from pypy.jit.metainterp.history import AbstractDescr from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.translator.unsimplify import varoftype @@ -145,3 +146,14 @@ int_add %i0, %i0, %i1 int_return %i1 """, transform=True) + + def test_descr(self): + class FooDescr(AbstractDescr): + def __repr__(self): + return 'hi_there!' + op = SpaceOperation('foobar', [FooDescr()], None) + flattener = GraphFlattener(None, fake_regallocs()) + flattener.serialize_op(op) + self.assert_format(flattener.ssarepr, """ + foobar hi_there! + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Fri Apr 23 16:50:16 2010 @@ -3,6 +3,7 @@ from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.flatten import Label, TLabel, SSARepr, Register from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.metainterp.history import AbstractDescr from pypy.rpython.lltypesystem import lltype @@ -67,3 +68,17 @@ foobar i[%i0, $123, %i1] """ assert asm == str(py.code.Source(expected)).strip() + '\n' + +def test_format_assembler_descr(): + class FooDescr(AbstractDescr): + def __repr__(self): + return 'hi_there!' + ssarepr = SSARepr("test") + ssarepr.insns = [ + ('foobar', FooDescr()), + ] + asm = format_assembler(ssarepr) + expected = """ + foobar hi_there! + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' From arigo at codespeak.net Fri Apr 23 16:54:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Apr 2010 16:54:09 +0200 (CEST) Subject: [pypy-svn] r74023 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100423145409.86D84282BD4@codespeak.net> Author: arigo Date: Fri Apr 23 16:54:07 2010 New Revision: 74023 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Log: AbstractDescrs in the assembler. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Fri Apr 23 16:54:07 2010 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.history import AbstractValue, getkind +from pypy.jit.metainterp.history import AbstractValue, AbstractDescr, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS from pypy.jit.codewriter.flatten import ListOfKind from pypy.objspace.flow.model import Constant @@ -29,6 +29,8 @@ def __init__(self): self.insns = {} + self.descrs = [] + self._descr_dict = {} def assemble(self, ssarepr): self.setup() @@ -108,6 +110,15 @@ raise NotImplementedError("found in ListOfKind(): %r" % (item,)) argcodes.append(itemkind[0].upper()) + elif isinstance(x, AbstractDescr): + if x not in self._descr_dict: + self._descr_dict[x] = len(self.descrs) + self.descrs.append(x) + num = self._descr_dict[x] + assert 0 <= num <= 0xFFFF, "too many AbstractDescrs!" + self.code.append(chr(num & 0xFF)) + self.code.append(chr(num >> 8)) + argcodes.append('d') else: raise NotImplementedError(x) # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Fri Apr 23 16:54:07 2010 @@ -1,6 +1,8 @@ +import struct from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.metainterp.history import AbstractDescr from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype @@ -93,3 +95,16 @@ assert jitcode.code == "\x00\x03\x16\x17\xFF\x00" assert assembler.insns == {'foobar/IR': 0} assert jitcode.constants_i == [42] + +def test_assemble_descr(): + class FooDescr(AbstractDescr): + pass + descrs = [FooDescr() for i in range(300)] + ssarepr = SSARepr("test") + ssarepr.insns = [('foobar', d) for d in descrs[::-1]] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ''.join(["\x00" + struct.pack(" Author: fijal Date: Fri Apr 23 16:59:55 2010 New Revision: 74024 Modified: pypy/trunk/pypy/module/termios/test/test_termios.py Log: fix the test Modified: pypy/trunk/pypy/module/termios/test/test_termios.py ============================================================================== --- pypy/trunk/pypy/module/termios/test/test_termios.py (original) +++ pypy/trunk/pypy/module/termios/test/test_termios.py Fri Apr 23 16:59:55 2010 @@ -130,4 +130,4 @@ def test_error_tcsetattr(self): import termios - raises(TypeError, termios.tcsetattr, 0, 1, (1, 2)) + raises(ValueError, termios.tcsetattr, 0, 1, (1, 2)) From fijal at codespeak.net Fri Apr 23 17:02:05 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 17:02:05 +0200 (CEST) Subject: [pypy-svn] r74025 - pypy/trunk/pypy/module/_socket/test Message-ID: <20100423150205.4F86C282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 17:02:03 2010 New Revision: 74025 Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py Log: Those things can also raise ValueError Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/trunk/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/trunk/pypy/module/_socket/test/test_sock_app.py Fri Apr 23 17:02:03 2010 @@ -344,7 +344,7 @@ import _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) for args in tests: - raises(TypeError, s.connect, args) + raises((TypeError, ValueError), s.connect, args) s.close() def test_NtoH(self): From fijal at codespeak.net Fri Apr 23 18:45:12 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 18:45:12 +0200 (CEST) Subject: [pypy-svn] r74026 - pypy/branch/rpython-iterator Message-ID: <20100423164512.8242D282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 18:45:11 2010 New Revision: 74026 Added: pypy/branch/rpython-iterator/ (props changed) - copied from r74025, pypy/trunk/ Log: A branch to try rpython iterator From fijal at codespeak.net Fri Apr 23 18:46:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 18:46:16 +0200 (CEST) Subject: [pypy-svn] r74027 - in pypy/branch/rpython-iterator/pypy: annotation rpython/lltypesystem Message-ID: <20100423164616.3CBAC282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 18:46:14 2010 New Revision: 74027 Modified: pypy/branch/rpython-iterator/pypy/annotation/unaryop.py pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py Log: IN-PROGRESS what I wrote so far Modified: pypy/branch/rpython-iterator/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/annotation/unaryop.py (original) +++ pypy/branch/rpython-iterator/pypy/annotation/unaryop.py Fri Apr 23 18:46:14 2010 @@ -11,7 +11,7 @@ s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ SomeGenericCallable, SomeWeakRef, SomeUnicodeString -from pypy.annotation.bookkeeper import getbookkeeper +from pypy.annotation.bookkeeper import getbookkeeper, RPythonCallsSpace from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? from pypy.rpython import extregistry @@ -584,30 +584,33 @@ class __extend__(SomeInstance): + def _lookup_const_attr(ins, attr): + if attr == '__class__': + return ins.classdef.read_attr__class__() + attrdef = ins.classdef.find_attribute(attr) + position = getbookkeeper().position_key + attrdef.read_locations[position] = True + s_result = attrdef.getvalue() + # hack: if s_result is a set of methods, discard the ones + # that can't possibly apply to an instance of ins.classdef. + # XXX do it more nicely + if isinstance(s_result, SomePBC): + s_result = ins.classdef.lookup_filter(s_result, attr, + ins.flags) + elif isinstance(s_result, SomeImpossibleValue): + ins.classdef.check_missing_attribute_update(attr) + # blocking is harmless if the attribute is explicitly listed + # in the class or a parent class. + for basedef in ins.classdef.getmro(): + if basedef.classdesc.all_enforced_attrs is not None: + if attr in basedef.classdesc.all_enforced_attrs: + raise HarmlesslyBlocked("get enforced attr") + return s_result + def getattr(ins, s_attr): if s_attr.is_constant() and isinstance(s_attr.const, str): attr = s_attr.const - if attr == '__class__': - return ins.classdef.read_attr__class__() - attrdef = ins.classdef.find_attribute(attr) - position = getbookkeeper().position_key - attrdef.read_locations[position] = True - s_result = attrdef.getvalue() - # hack: if s_result is a set of methods, discard the ones - # that can't possibly apply to an instance of ins.classdef. - # XXX do it more nicely - if isinstance(s_result, SomePBC): - s_result = ins.classdef.lookup_filter(s_result, attr, - ins.flags) - elif isinstance(s_result, SomeImpossibleValue): - ins.classdef.check_missing_attribute_update(attr) - # blocking is harmless if the attribute is explicitly listed - # in the class or a parent class. - for basedef in ins.classdef.getmro(): - if basedef.classdesc.all_enforced_attrs is not None: - if attr in basedef.classdesc.all_enforced_attrs: - raise HarmlesslyBlocked("get enforced attr") - return s_result + return ins._lookup_const_attr(attr) return SomeObject() getattr.can_only_throw = [] @@ -629,6 +632,16 @@ if not ins.can_be_None: s.const = True + def iter(ins): + s_result = SomeIterator(ins) + s_result.s_next_pbc = ins._lookup_const_attr('next') + return s_result + + def getanyitem(ins): + bk = getbookkeeper() + args = bk.build_args("simple_call", []) + s_pbc = ins._lookup_const_attr('next') + return bk.pbc_call(s_pbc, args) class __extend__(SomeBuiltin): def simple_call(bltn, *args): Modified: pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py Fri Apr 23 18:46:14 2010 @@ -587,7 +587,19 @@ else: return hop.gendirectcall(ll_isinstance, v_obj, v_cls) + def make_iterator_repr(self): + return self # I'm the iterator repr + def newiter(self, hop): + vinst, = hop.inputargs(self) + return vinst + + def rtype_next(self, hop): + vinst, = hop.inputargs(self) + r_method = self.rtyper.makerepr(hop.args_s[0].s_next_pbc) + v_meth = r_method.get_method_from_instance(self, vinst, hop.llops) + import pdb + pdb.set_trace() class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): From fijal at codespeak.net Fri Apr 23 19:11:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 19:11:23 +0200 (CEST) Subject: [pypy-svn] r74028 - in pypy/branch/rpython-iterator/pypy/rlib: . test Message-ID: <20100423171123.8106A282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 19:11:22 2010 New Revision: 74028 Added: pypy/branch/rpython-iterator/pypy/rlib/r_iter.py (contents, props changed) pypy/branch/rpython-iterator/pypy/rlib/test/test_r_iter.py (contents, props changed) Log: misssing files Added: pypy/branch/rpython-iterator/pypy/rlib/r_iter.py ============================================================================== --- (empty file) +++ pypy/branch/rpython-iterator/pypy/rlib/r_iter.py Fri Apr 23 19:11:22 2010 @@ -0,0 +1,24 @@ + +from pypy.rpython.extregistry import ExtRegistryEntry + +class r_iter(object): + def next(self): + raise NotImplementedError("abstract base class") + +class list_iter(object): + def __init__(self, l): + self.l = l + self.pos = 0 + + def next(self): + if self.pos >= len(self.l): + raise StopIteration + res = self.l[self.pos] + self.pos += 1 + return res + + def __iter__(self): + """ NOT_RPYTHON, for untranslated version only + """ + return self + Added: pypy/branch/rpython-iterator/pypy/rlib/test/test_r_iter.py ============================================================================== --- (empty file) +++ pypy/branch/rpython-iterator/pypy/rlib/test/test_r_iter.py Fri Apr 23 19:11:22 2010 @@ -0,0 +1,19 @@ + +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin +from pypy.rlib.r_iter import list_iter + +class AbstractRIterTest(BaseRtypingTest): + def test_basic(self): + def f(): + l = [1, 2, 3, 4, 5] + a = [] + for i in list_iter(l): + a.append(i) + return len(a) + + assert f() == 5 + res = self.interpret(f, []) + assert res == 5 + +class TestRType(AbstractRIterTest, LLRtypeMixin): + pass From arigo at codespeak.net Fri Apr 23 19:20:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 23 Apr 2010 19:20:20 +0200 (CEST) Subject: [pypy-svn] r74029 - in pypy/branch/rpython-iterator/pypy: annotation rpython/lltypesystem Message-ID: <20100423172020.1C1F4282BD4@codespeak.net> Author: arigo Date: Fri Apr 23 19:20:18 2010 New Revision: 74029 Modified: pypy/branch/rpython-iterator/pypy/annotation/unaryop.py pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py Log: Tweaks and copy-pasting from controllerentry.py until the basic test in rlib/test/test_r_iter works. Modified: pypy/branch/rpython-iterator/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/annotation/unaryop.py (original) +++ pypy/branch/rpython-iterator/pypy/annotation/unaryop.py Fri Apr 23 19:20:18 2010 @@ -11,7 +11,7 @@ s_ImpossibleValue, s_Bool, s_None, \ unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ SomeGenericCallable, SomeWeakRef, SomeUnicodeString -from pypy.annotation.bookkeeper import getbookkeeper, RPythonCallsSpace +from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? from pypy.rpython import extregistry @@ -633,15 +633,17 @@ s.const = True def iter(ins): - s_result = SomeIterator(ins) - s_result.s_next_pbc = ins._lookup_const_attr('next') - return s_result + return SomeIterator(ins) def getanyitem(ins): bk = getbookkeeper() - args = bk.build_args("simple_call", []) - s_pbc = ins._lookup_const_attr('next') - return bk.pbc_call(s_pbc, args) + s_func = bk.immutablevalue(call_next) + return bk.emulate_pbc_call(bk.position_key, s_func, [ins], + callback = bk.position_key) + +# XXX temporary! +def call_next(instance): + return instance.next() class __extend__(SomeBuiltin): def simple_call(bltn, *args): Modified: pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/rpython-iterator/pypy/rpython/lltypesystem/rclass.py Fri Apr 23 19:20:18 2010 @@ -595,11 +595,16 @@ return vinst def rtype_next(self, hop): - vinst, = hop.inputargs(self) - r_method = self.rtyper.makerepr(hop.args_s[0].s_next_pbc) - v_meth = r_method.get_method_from_instance(self, vinst, hop.llops) - import pdb - pdb.set_trace() + # XXX should be moved to rpython/rclass.py + bk = hop.rtyper.annotator.bookkeeper + hop2 = hop.copy() + # XXX fish fish fish + from pypy.annotation.unaryop import call_next + c_func = Constant(call_next) + s_func = bk.immutablevalue(call_next) + hop2.v_s_insertfirstarg(c_func, s_func) + hop2.forced_opname = 'simple_call' + return hop2.dispatch() class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): From fijal at codespeak.net Fri Apr 23 19:54:29 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 19:54:29 +0200 (CEST) Subject: [pypy-svn] r74030 - pypy/branch/rpython-iterator/pypy/rlib Message-ID: <20100423175429.9B05E282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 19:54:27 2010 New Revision: 74030 Modified: pypy/branch/rpython-iterator/pypy/rlib/r_iter.py Log: Fix inheritance hierarchy. I'll fry in hell for not writing tests Modified: pypy/branch/rpython-iterator/pypy/rlib/r_iter.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/rlib/r_iter.py (original) +++ pypy/branch/rpython-iterator/pypy/rlib/r_iter.py Fri Apr 23 19:54:27 2010 @@ -5,7 +5,12 @@ def next(self): raise NotImplementedError("abstract base class") -class list_iter(object): + def __iter__(self): + """ NOT_RPYTHON, for untranslated version only + """ + return self + +class list_iter(r_iter): def __init__(self, l): self.l = l self.pos = 0 @@ -16,9 +21,4 @@ res = self.l[self.pos] self.pos += 1 return res - - def __iter__(self): - """ NOT_RPYTHON, for untranslated version only - """ - return self From fijal at codespeak.net Fri Apr 23 19:54:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 19:54:55 +0200 (CEST) Subject: [pypy-svn] r74031 - pypy/branch/rpython-iterator/pypy/objspace/std Message-ID: <20100423175455.3F70E282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 19:54:53 2010 New Revision: 74031 Modified: pypy/branch/rpython-iterator/pypy/objspace/std/objspace.py pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py Log: A quick hack to measure performance - use newly created iterator for unicode join Modified: pypy/branch/rpython-iterator/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/objspace.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/objspace.py Fri Apr 23 19:54:53 2010 @@ -36,7 +36,21 @@ from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.stringtype import wrapstr from pypy.objspace.std.unicodetype import wrapunicode +from pypy.rlib.r_iter import r_iter, list_iter +class iterate_w_root(r_iter): + def __init__(self, space, w_iterable): + self.w_iterator = space.iter(w_iterable) + self.space = space + + def next(self): + space = self.space + try: + return space.next(self.w_iterator) + except OperationError, oe: + if oe.match(space, space.w_StopIteration): + raise StopIteration + raise class StdObjSpace(ObjSpace, DescrOperation): """The standard object space, implementing a general-purpose object @@ -347,6 +361,14 @@ raise self._wrap_expected_length(expected_length, len(t)) return t + def iterate_w(self, w_iterable): + """ Returns a thing that can be iterated over. No more + operations are allowed, primarily, this is not a list + """ + if isinstance(w_iterable, W_ListObject): + return list_iter(w_iterable.wrappeditems) + return iterate_w_root(self, w_iterable) + def fixedview(self, w_obj, expected_length=-1): """ Fast paths """ Modified: pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py Fri Apr 23 19:54:53 2010 @@ -181,27 +181,23 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.unpackiterable(w_list) - delim = w_self._value - totlen = 0 - if len(l) == 0: - return W_UnicodeObject.EMPTY - if (len(l) == 1 and - space.is_w(space.type(l[0]), space.w_unicode)): - return l[0] + #if len(l) == 0: + # return W_UnicodeObject.EMPTY + #if (len(l) == 1 and + # space.is_w(space.type(l[0]), space.w_unicode)): + # return l[0] - values_list = [None] * len(l) - for i in range(len(l)): - item = l[i] - if isinstance(item, W_UnicodeObject): + values_list = []#None] * len(l) + for w_item in space.iterate_w(w_list): + if isinstance(w_item, W_UnicodeObject): # shortcut for performane - item = item._value - elif space.is_true(space.isinstance(item, space.w_str)): - item = space.unicode_w(item) + item = w_item._value + elif space.is_true(space.isinstance(w_item, space.w_str)): + item = space.unicode_w(w_item) else: raise operationerrfmt(space.w_TypeError, "sequence item %d: expected string or Unicode", i) - values_list[i] = item + values_list.append(item) return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): From fijal at codespeak.net Fri Apr 23 20:00:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 20:00:53 +0200 (CEST) Subject: [pypy-svn] r74032 - in pypy/branch/rpython-iterator/pypy/objspace/std: . test Message-ID: <20100423180053.09716282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 20:00:51 2010 New Revision: 74032 Modified: pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py Log: A fix and a test! (untested path) Modified: pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py Fri Apr 23 20:00:51 2010 @@ -64,6 +64,7 @@ check(', '.join([u'a']), u'a') check(', '.join(['a', u'b']), u'a, b') check(u', '.join(['a', 'b']), u'a, b') + raises(TypeError, u'x'.join, [u'x', (1, 2, 3)]) if sys.version_info >= (2,3): def test_contains_ex(self): Modified: pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py Fri Apr 23 20:00:51 2010 @@ -188,7 +188,9 @@ # return l[0] values_list = []#None] * len(l) + i = 0 for w_item in space.iterate_w(w_list): + i += 1 if isinstance(w_item, W_UnicodeObject): # shortcut for performane item = w_item._value From fijal at codespeak.net Fri Apr 23 21:29:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Apr 2010 21:29:33 +0200 (CEST) Subject: [pypy-svn] r74033 - in pypy/branch/rpython-iterator/pypy: objspace/std objspace/std/test rlib Message-ID: <20100423192933.4BB2E282BD4@codespeak.net> Author: fijal Date: Fri Apr 23 21:29:31 2010 New Revision: 74033 Modified: pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py pypy/branch/rpython-iterator/pypy/rlib/r_iter.py Log: Implement length_hint (that can return 0), let's see if this speeds stuff up Modified: pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/test/test_unicodeobject.py Fri Apr 23 21:29:31 2010 @@ -66,6 +66,19 @@ check(u', '.join(['a', 'b']), u'a, b') raises(TypeError, u'x'.join, [u'x', (1, 2, 3)]) + class X(object): + def __init__(self, num): + self.num = num + def __iter__(self): + return self + def next(self): + if self.num == 0: + raise StopIteration + self.num -= 1 + return u"abc" + + check(u''.join(X(3)), u"abcabcabc") + if sys.version_info >= (2,3): def test_contains_ex(self): assert u'' in 'abc' Modified: pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/rpython-iterator/pypy/objspace/std/unicodeobject.py Fri Apr 23 21:29:31 2010 @@ -187,10 +187,14 @@ # space.is_w(space.type(l[0]), space.w_unicode)): # return l[0] - values_list = []#None] * len(l) i = 0 - for w_item in space.iterate_w(w_list): - i += 1 + iterator = space.iterate_w(w_list) + values_list = [None] * iterator.length_hint() + if values_list: + append = False + else: + append = True + for w_item in iterator: if isinstance(w_item, W_UnicodeObject): # shortcut for performane item = w_item._value @@ -199,7 +203,11 @@ else: raise operationerrfmt(space.w_TypeError, "sequence item %d: expected string or Unicode", i) - values_list.append(item) + if append: + values_list.append(item) + else: + values_list[i] = item + i += 1 return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): Modified: pypy/branch/rpython-iterator/pypy/rlib/r_iter.py ============================================================================== --- pypy/branch/rpython-iterator/pypy/rlib/r_iter.py (original) +++ pypy/branch/rpython-iterator/pypy/rlib/r_iter.py Fri Apr 23 21:29:31 2010 @@ -10,6 +10,9 @@ """ return self + def length_hint(self): + return 0 + class list_iter(r_iter): def __init__(self, l): self.l = l @@ -21,4 +24,6 @@ res = self.l[self.pos] self.pos += 1 return res - + + def length_hint(self): + return len(self.l) From arigo at codespeak.net Sat Apr 24 17:29:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 17:29:50 +0200 (CEST) Subject: [pypy-svn] r74034 - in pypy/trunk: lib-python/modified-2.5.2/test pypy/module/_socket pypy/module/_socket/test Message-ID: <20100424152950.9B9DF282B90@codespeak.net> Author: arigo Date: Sat Apr 24 17:29:48 2010 New Revision: 74034 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py pypy/trunk/pypy/module/_socket/interp_func.py pypy/trunk/pypy/module/_socket/test/test_sock_app.py Log: A fix for 64-bit, including more corner cases that are wrong in various versions of CPython. Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py Sat Apr 24 17:29:48 2010 @@ -307,10 +307,23 @@ mask = (1L<= 2.6: + self.assertRaises(OverflowError, func, -1) def testGetServBy(self): eq = self.assertEqual Modified: pypy/trunk/pypy/module/_socket/interp_func.py ============================================================================== --- pypy/trunk/pypy/module/_socket/interp_func.py (original) +++ pypy/trunk/pypy/module/_socket/interp_func.py Sat Apr 24 17:29:48 2010 @@ -162,7 +162,7 @@ Convert a 16-bit integer from network to host byte order. """ return space.wrap(rsocket.ntohs(x)) -ntohs.unwrap_spec = [ObjSpace, r_uint] +ntohs.unwrap_spec = [ObjSpace, "c_uint"] def ntohl(space, x): """ntohl(integer) -> integer @@ -170,7 +170,7 @@ Convert a 32-bit integer from network to host byte order. """ return space.wrap(rsocket.ntohl(x)) -ntohl.unwrap_spec = [ObjSpace, r_uint] +ntohl.unwrap_spec = [ObjSpace, "c_uint"] def htons(space, x): """htons(integer) -> integer @@ -178,7 +178,7 @@ Convert a 16-bit integer from host to network byte order. """ return space.wrap(rsocket.htons(x)) -htons.unwrap_spec = [ObjSpace, r_uint] +htons.unwrap_spec = [ObjSpace, "c_uint"] def htonl(space, x): """htonl(integer) -> integer @@ -186,7 +186,7 @@ Convert a 32-bit integer from host to network byte order. """ return space.wrap(rsocket.htonl(x)) -htonl.unwrap_spec = [ObjSpace, r_uint] +htonl.unwrap_spec = [ObjSpace, "c_uint"] def inet_aton(space, ip): """inet_aton(string) -> packed 32-bit IP representation Modified: pypy/trunk/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/trunk/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/trunk/pypy/module/_socket/test/test_sock_app.py Sat Apr 24 17:29:48 2010 @@ -115,6 +115,9 @@ w_n = space.appexec([w_socket, space.wrap(0x89abcdef)], "(_socket, x): return _socket.ntohl(x)") assert space.unwrap(w_n) in (0x89abcdef, 0xefcdab89) + space.raises_w(space.w_OverflowError, space.appexec, + [w_socket, space.wrap(1<<32)], + "(_socket, x): return _socket.ntohl(x)") def test_htons(): w_n = space.appexec([w_socket, space.wrap(125)], @@ -128,6 +131,9 @@ w_n = space.appexec([w_socket, space.wrap(0x89abcdef)], "(_socket, x): return _socket.htonl(x)") assert space.unwrap(w_n) in (0x89abcdef, 0xefcdab89) + space.raises_w(space.w_OverflowError, space.appexec, + [w_socket, space.wrap(1<<32)], + "(_socket, x): return _socket.htonl(x)") def test_aton_ntoa(): ip = '123.45.67.89' From arigo at codespeak.net Sat Apr 24 17:38:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 17:38:41 +0200 (CEST) Subject: [pypy-svn] r74035 - pypy/trunk/pypy/module/posix/test Message-ID: <20100424153841.11035282B90@codespeak.net> Author: arigo Date: Sat Apr 24 17:38:39 2010 New Revision: 74035 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: Don't crash with an AttributeError if there was a problem running setup(). Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Sat Apr 24 17:38:39 2010 @@ -658,7 +658,10 @@ space.sys.filesystemencoding = "utf-8" def teardown_class(cls): - cls.space.sys.filesystemencoding = cls.save_fs_encoding + try: + cls.space.sys.filesystemencoding = cls.save_fs_encoding + except AttributeError: + pass def test_stat_unicode(self): # test that passing unicode would not raise UnicodeDecodeError From arigo at codespeak.net Sat Apr 24 18:00:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 18:00:17 +0200 (CEST) Subject: [pypy-svn] r74036 - in pypy/branch/blackhole-improvement/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c translator/c/test Message-ID: <20100424160017.DAC6D282B90@codespeak.net> Author: arigo Date: Sat Apr 24 18:00:16 2010 New Revision: 74036 Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/branch/blackhole-improvement/pypy/translator/c/primitive.py pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py Log: Change llmemory.cast_adr_to_int() to return a Symbolic. This Symbolic can be cast back to an address with llmemory.cast_int_to_adr(). The advantage is that it can be supported as a prebuilt constant translated to C. Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Sat Apr 24 18:00:16 2010 @@ -479,10 +479,8 @@ return lltype.nullptr(EXPECTED_TYPE.TO) def _cast_to_int(self): - # This is a bit annoying. We want this method to still work when the - # pointed-to object is dead if self: - return self.ptr._cast_to_int(False) + return AddressAsInt(self) else: return 0 @@ -496,6 +494,18 @@ # ____________________________________________________________ +class AddressAsInt(Symbolic): + # a symbolic, rendered as an address cast to an integer. + def __init__(self, adr): + self.adr = adr + def annotation(self): + from pypy.annotation import model + return model.SomeInteger() + def lltype(self): + return lltype.Signed + +# ____________________________________________________________ + class NullAddressError(Exception): pass @@ -608,6 +618,8 @@ _NONGCREF = lltype.Ptr(lltype.OpaqueType('NONGCREF')) def cast_int_to_adr(int): + if isinstance(int, AddressAsInt): + return int.adr ptr = lltype.cast_int_to_ptr(_NONGCREF, int) return cast_ptr_to_adr(ptr) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/test/test_llmemory.py Sat Apr 24 18:00:16 2010 @@ -624,3 +624,14 @@ # the following line crashes if the array is dead ptr1 = cast_adr_to_ptr(adr, lltype.Ptr(lltype.FixedSizeArray(Address, 1))) ptr1[0] = NULL + +def test_cast_adr_to_int(): + A = lltype.Array(Address) + ptr = lltype.malloc(A, 10, immortal=True) + adr = cast_ptr_to_adr(ptr) + int = cast_adr_to_int(adr) + assert isinstance(int, AddressAsInt) + assert cast_int_to_adr(int) == adr + # + assert cast_adr_to_int(NULL) == 0 + assert cast_int_to_adr(0) == NULL Modified: pypy/branch/blackhole-improvement/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/c/primitive.py (original) +++ pypy/branch/blackhole-improvement/pypy/translator/c/primitive.py Sat Apr 24 18:00:16 2010 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset, GCREF + GCHeaderOffset, GCREF, AddressAsInt from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation from pypy.translator.c.support import cdecl, barebonearray @@ -61,6 +61,8 @@ name = name_small_integer(value.lowpart, db) assert (value.rest & value.MASK) == 0 return '(%s+%dL)' % (name, value.rest) + elif isinstance(value, AddressAsInt): + return '((long)%s)' % name_address(value.adr, db) else: raise Exception("unimplemented symbolic %r"%value) if value is None: Modified: pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/blackhole-improvement/pypy/translator/c/test/test_lladdresses.py Sat Apr 24 18:00:16 2010 @@ -193,3 +193,21 @@ fc = compile(f, [int]) res = fc(42) assert res == 0 + +def test_cast_int_to_adr(): + S = lltype.Struct("S", ("x", lltype.Signed)) + s = lltype.malloc(S, immortal=True) + s.x = 42 + integer = cast_adr_to_int(cast_ptr_to_adr(s)) + def f(n): + if n > 1: + i = integer + else: + i = 123 # nonsense, but never used + print "hello world" # prevent constant-folding + adr = cast_int_to_adr(i) + s = cast_adr_to_ptr(adr, lltype.Ptr(S)) + return s.x + fc = compile(f, [int]) + res = fc(5) + assert res == 42 From arigo at codespeak.net Sat Apr 24 18:13:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 18:13:34 +0200 (CEST) Subject: [pypy-svn] r74037 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100424161334.346C2282B90@codespeak.net> Author: arigo Date: Sat Apr 24 18:13:32 2010 New Revision: 74037 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Log: Correctly cast incoming constants to either Signed, GCREF or Float. Non-GC pointers end up being cast via Address to 'AddressAsInt', a Symbolic Signed. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Sat Apr 24 18:13:32 2010 @@ -2,6 +2,7 @@ from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS from pypy.jit.codewriter.flatten import ListOfKind from pypy.objspace.flow.model import Constant +from pypy.rpython.lltypesystem import lltype, llmemory class JitCode(AbstractValue): @@ -55,19 +56,38 @@ self.highest_regs[reg.kind] = reg.index self.code.append(chr(reg.index)) - def emit_const(self, const, kind): + def emit_const(self, const, kind, allow_short=False): if const not in self.constants_dict: + value = const.value + TYPE = lltype.typeOf(value) if kind == 'int': + if isinstance(TYPE, lltype.Ptr): + assert TYPE.TO._gckind == 'raw' + value = llmemory.cast_ptr_to_adr(value) + TYPE = llmemory.Address + if TYPE == llmemory.Address: + value = llmemory.cast_adr_to_int(value) + else: + value = lltype.cast_primitive(lltype.Signed, value) + if allow_short and -128 <= value <= 127: # xxx symbolic + # emit the constant as a small integer + self.code.append(chr(value & 0xFF)) + return True constants = self.constants_i elif kind == 'ref': + value = lltype.cast_opaque_ptr(llmemory.GCREF, value) constants = self.constants_r elif kind == 'float': + assert TYPE == lltype.Float constants = self.constants_f else: raise NotImplementedError(const) - constants.append(const.value) + constants.append(value) self.constants_dict[const] = 256 - len(constants) + # emit the constant normally, as one byte that is an index in the + # list of constants self.code.append(chr(self.constants_dict[const])) + return False def write_insn(self, insn): if isinstance(insn[0], Label): @@ -83,11 +103,10 @@ argcodes.append(x.kind[0]) elif isinstance(x, Constant): kind = getkind(x.concretetype) - if kind == 'int' and -128 <= x.value <= 127: - self.code.append(chr(x.value & 0xFF)) + is_short = self.emit_const(x, kind, allow_short=True) + if is_short: argcodes.append('c') else: - self.emit_const(x, kind) argcodes.append(kind[0]) elif isinstance(x, TLabel): self.tlabel_positions.append((x.name, len(self.code))) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Sat Apr 24 18:13:32 2010 @@ -4,7 +4,7 @@ from pypy.jit.codewriter.flatten import ListOfKind from pypy.jit.metainterp.history import AbstractDescr from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory def test_assemble_simple(): @@ -58,6 +58,32 @@ assert assembler.insns == {'float_return/f': 0} assert jitcode.constants_f == [18.0, -4.0, 128.1] +def test_assemble_cast_consts(): + ssarepr = SSARepr("test") + S = lltype.GcStruct('S') + s = lltype.malloc(S) + F = lltype.FuncType([], lltype.Signed) + f = lltype.functionptr(F, 'f') + ssarepr.insns = [ + ('int_return', Constant('X', lltype.Char)), + ('int_return', Constant(unichr(0x1234), lltype.UniChar)), + ('int_return', Constant(f, lltype.Ptr(F))), + ('ref_return', Constant(s, lltype.Ptr(S))), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode.code == ("\x00\x58" + "\x01\xFF" + "\x01\xFE" + "\x02\xFF") + assert assembler.insns == {'int_return/c': 0, + 'int_return/i': 1, + 'ref_return/r': 2} + f_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(f)) + assert jitcode.constants_i == [0x1234, f_int] + s_gcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) + assert jitcode.constants_r == [s_gcref] + def test_assemble_loop(): ssarepr = SSARepr("test") i0, i1 = Register('int', 0x16), Register('int', 0x17) From arigo at codespeak.net Sat Apr 24 18:40:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 18:40:02 +0200 (CEST) Subject: [pypy-svn] r74038 - pypy/pysqlite2/test Message-ID: <20100424164002.7D43F282B90@codespeak.net> Author: arigo Date: Sat Apr 24 18:40:00 2010 New Revision: 74038 Modified: pypy/pysqlite2/test/dbapi.py pypy/pysqlite2/test/factory.py pypy/pysqlite2/test/hooks.py pypy/pysqlite2/test/py25tests.py pypy/pysqlite2/test/regression.py pypy/pysqlite2/test/transactions.py pypy/pysqlite2/test/types.py pypy/pysqlite2/test/userfunctions.py Log: Increase the verbosity of these tests. Modified: pypy/pysqlite2/test/dbapi.py ============================================================================== --- pypy/pysqlite2/test/dbapi.py (original) +++ pypy/pysqlite2/test/dbapi.py Sat Apr 24 18:40:00 2010 @@ -812,7 +812,7 @@ return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, ext_suite, closed_suite, bug_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/factory.py ============================================================================== --- pypy/pysqlite2/test/factory.py (original) +++ pypy/pysqlite2/test/factory.py Sat Apr 24 18:40:00 2010 @@ -178,7 +178,7 @@ return unittest.TestSuite((connection_suite, cursor_suite, row_suite_compat, row_suite, text_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/hooks.py ============================================================================== --- pypy/pysqlite2/test/hooks.py (original) +++ pypy/pysqlite2/test/hooks.py Sat Apr 24 18:40:00 2010 @@ -189,7 +189,7 @@ return unittest.TestSuite((collation_suite, progress_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/py25tests.py ============================================================================== --- pypy/pysqlite2/test/py25tests.py (original) +++ pypy/pysqlite2/test/py25tests.py Sat Apr 24 18:40:00 2010 @@ -73,7 +73,7 @@ return unittest.TestSuite((ctx_suite,)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/regression.py ============================================================================== --- pypy/pysqlite2/test/regression.py (original) +++ pypy/pysqlite2/test/regression.py Sat Apr 24 18:40:00 2010 @@ -141,7 +141,7 @@ return unittest.TestSuite((regression_suite,)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/transactions.py ============================================================================== --- pypy/pysqlite2/test/transactions.py (original) +++ pypy/pysqlite2/test/transactions.py Sat Apr 24 18:40:00 2010 @@ -167,7 +167,7 @@ return unittest.TestSuite((default_suite, special_command_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/types.py ============================================================================== --- pypy/pysqlite2/test/types.py (original) +++ pypy/pysqlite2/test/types.py Sat Apr 24 18:40:00 2010 @@ -338,7 +338,7 @@ return unittest.TestSuite((sqlite_type_suite, decltypes_type_suite, colnames_type_suite, adaptation_suite, bin_suite, date_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": Modified: pypy/pysqlite2/test/userfunctions.py ============================================================================== --- pypy/pysqlite2/test/userfunctions.py (original) +++ pypy/pysqlite2/test/userfunctions.py Sat Apr 24 18:40:00 2010 @@ -408,7 +408,7 @@ return unittest.TestSuite((function_suite, aggregate_suite, authorizer_suite)) def test(): - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(verbosity=2) runner.run(suite()) if __name__ == "__main__": From arigo at codespeak.net Sat Apr 24 18:40:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 18:40:16 +0200 (CEST) Subject: [pypy-svn] r74039 - pypy/pysqlite2 Message-ID: <20100424164016.3D6A3282B90@codespeak.net> Author: arigo Date: Sat Apr 24 18:40:14 2010 New Revision: 74039 Modified: pypy/pysqlite2/dbapi2.py Log: Fix on 64-bit platforms. Modified: pypy/pysqlite2/dbapi2.py ============================================================================== --- pypy/pysqlite2/dbapi2.py (original) +++ pypy/pysqlite2/dbapi2.py Sat Apr 24 18:40:14 2010 @@ -106,7 +106,7 @@ SQLITE_TEXT = 3 SQLITE3_TEXT = 3 -SQLITE_TRANSIENT = -1 +SQLITE_TRANSIENT = cast(-1, c_void_p) SQLITE_UTF8 = 1 SQLITE_DENY = 1 @@ -173,8 +173,8 @@ sqlite.sqlite3_bind_parameter_index.restype = c_int sqlite.sqlite3_bind_parameter_name.argtypes = [c_void_p, c_int] sqlite.sqlite3_bind_parameter_name.restype = c_char_p -sqlite.sqlite3_bind_text.argtypes = [c_void_p, c_int, c_char_p, c_int, c_int] -sqlite.sqlite3_bind_blob.argtypes = [c_void_p, c_int, c_void_p, c_int, c_int] +sqlite.sqlite3_bind_text.argtypes = [c_void_p, c_int, c_char_p, c_int,c_void_p] +sqlite.sqlite3_bind_blob.argtypes = [c_void_p, c_int, c_void_p, c_int,c_void_p] sqlite.sqlite3_bind_blob.restype = c_int sqlite.sqlite3_changes.argtypes = [c_void_p] sqlite.sqlite3_changes.restype = c_int From arigo at codespeak.net Sat Apr 24 18:44:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 24 Apr 2010 18:44:54 +0200 (CEST) Subject: [pypy-svn] r74040 - pypy/pysqlite2 Message-ID: <20100424164454.BC278282B90@codespeak.net> Author: arigo Date: Sat Apr 24 18:44:53 2010 New Revision: 74040 Modified: pypy/pysqlite2/dbapi2.py Log: Fix for test/types.py on 64-bits platforms. Modified: pypy/pysqlite2/dbapi2.py ============================================================================== --- pypy/pysqlite2/dbapi2.py (original) +++ pypy/pysqlite2/dbapi2.py Sat Apr 24 18:44:53 2010 @@ -682,10 +682,11 @@ if param is None: sqlite.sqlite3_bind_null(self.statement, idx) - elif type(param) in (bool, int): - sqlite.sqlite3_bind_int(self.statement, idx, param) - elif type(param) is long: - sqlite.sqlite3_bind_int64(self.statement, idx, param) + elif type(param) in (bool, int, long): + if -2147483648 <= param <= 2147483647: + sqlite.sqlite3_bind_int(self.statement, idx, param) + else: + sqlite.sqlite3_bind_int64(self.statement, idx, param) elif type(param) is float: sqlite.sqlite3_bind_double(self.statement, idx, param) elif isinstance(param, str): From benjamin at codespeak.net Sat Apr 24 23:36:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Apr 2010 23:36:35 +0200 (CEST) Subject: [pypy-svn] r74041 - pypy/trunk/pypy/annotation/test Message-ID: <20100424213635.2CE62282B90@codespeak.net> Author: benjamin Date: Sat Apr 24 23:36:30 2010 New Revision: 74041 Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py Log: test for staticmethod Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Sat Apr 24 23:36:30 2010 @@ -223,6 +223,17 @@ # result should be a list of integers assert listitem(s).knowntype == int + def test_staticmethod(self): + class X(object): + @staticmethod + def stat(value): + return value + 4 + def f(v): + return X().stat(v) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeInteger) + def test_methodcall1(self): a = self.RPythonAnnotator() s = a.build_types(snippet._methodcall1, [int]) From benjamin at codespeak.net Sat Apr 24 23:47:10 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Apr 2010 23:47:10 +0200 (CEST) Subject: [pypy-svn] r74042 - in pypy/trunk/pypy/annotation: . test Message-ID: <20100424214710.82510282B90@codespeak.net> Author: benjamin Date: Sat Apr 24 23:47:08 2010 New Revision: 74042 Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py Log: explode sooner rather than later on class methods Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sat Apr 24 23:47:08 2010 @@ -595,6 +595,8 @@ if isinstance(value, staticmethod): # special case value = value.__get__(42) classdef = None # don't bind + elif isinstance(value, classmethod): + raise AssertionError("classmethods are not supported") s_value = self.bookkeeper.immutablevalue(value) if classdef is not None: s_value = s_value.bind_callables_under(classdef, name) Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Sat Apr 24 23:47:08 2010 @@ -234,6 +234,16 @@ s = a.build_types(f, [int]) assert isinstance(s, annmodel.SomeInteger) + def test_classmethod(self): + class X(object): + @classmethod + def meth(cls): + return None + def f(): + return X().meth() + a = self.RPythonAnnotator() + py.test.raises(AssertionError, a.build_types, f, []) + def test_methodcall1(self): a = self.RPythonAnnotator() s = a.build_types(snippet._methodcall1, [int]) From benjamin at codespeak.net Sun Apr 25 03:18:35 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 25 Apr 2010 03:18:35 +0200 (CEST) Subject: [pypy-svn] r74043 - in pypy/trunk/pypy: annotation translator/backendopt Message-ID: <20100425011835.85ED9282B90@codespeak.net> Author: benjamin Date: Sun Apr 25 03:18:33 2010 New Revision: 74043 Modified: pypy/trunk/pypy/annotation/binaryop.py pypy/trunk/pypy/annotation/model.py pypy/trunk/pypy/annotation/unaryop.py pypy/trunk/pypy/translator/backendopt/support.py Log: use builtin sets Modified: pypy/trunk/pypy/annotation/binaryop.py ============================================================================== --- pypy/trunk/pypy/annotation/binaryop.py (original) +++ pypy/trunk/pypy/annotation/binaryop.py Sun Apr 25 03:18:33 2010 @@ -14,7 +14,7 @@ from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeSingleFloat -from pypy.annotation.model import unionof, UnionError, set, missing_operation +from pypy.annotation.model import unionof, UnionError, missing_operation from pypy.annotation.model import isdegenerated, TLS from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata Modified: pypy/trunk/pypy/annotation/model.py ============================================================================== --- pypy/trunk/pypy/annotation/model.py (original) +++ pypy/trunk/pypy/annotation/model.py Sun Apr 25 03:18:33 2010 @@ -695,19 +695,6 @@ # ____________________________________________________________ # internal -def setunion(d1, d2): - "Union of two sets represented as dictionaries." - d = d1.copy() - d.update(d2) - return d - -def set(it): - "Turn an iterable into a set." - d = {} - for x in it: - d[x] = True - return d - def commonbase(cls1, cls2): # XXX single inheritance only XXX hum l1 = inspect.getmro(cls1) l2 = inspect.getmro(cls2) Modified: pypy/trunk/pypy/annotation/unaryop.py ============================================================================== --- pypy/trunk/pypy/annotation/unaryop.py (original) +++ pypy/trunk/pypy/annotation/unaryop.py Sun Apr 25 03:18:33 2010 @@ -9,7 +9,7 @@ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ - unionof, set, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ + unionof, missing_operation, add_knowntypedata, HarmlesslyBlocked, \ SomeGenericCallable, SomeWeakRef, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import builtin Modified: pypy/trunk/pypy/translator/backendopt/support.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/support.py (original) +++ pypy/trunk/pypy/translator/backendopt/support.py Sun Apr 25 03:18:33 2010 @@ -3,7 +3,7 @@ from pypy.translator.simplify import get_graph from pypy.rpython.rmodel import inputconst from pypy.tool.ansi_print import ansi_log -from pypy.annotation.model import setunion, s_ImpossibleValue +from pypy.annotation.model import s_ImpossibleValue from pypy.translator.unsimplify import split_block, copyvar, insert_empty_block from pypy.objspace.flow.model import Constant, Variable, SpaceOperation, c_last_exception from pypy.rpython.lltypesystem import lltype @@ -152,17 +152,17 @@ reachable = {} blocks = list(graph.iterblocks()) for block in py.builtin.reversed(blocks): # this order should make the reuse path more likely - reach = {} + reach = set() scheduled = [block] while scheduled: current = scheduled.pop() for link in current.exits: if link.target in reachable: - reach[link.target] = True - reach = setunion(reach, reachable[link.target]) + reach.add(link.target) + reach = reach | reachable[link.target] continue if link.target not in reach: - reach[link.target] = True + reach.add(link.target) scheduled.append(link.target) reachable[block] = reach return reachable From benjamin at codespeak.net Sun Apr 25 03:53:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 25 Apr 2010 03:53:45 +0200 (CEST) Subject: [pypy-svn] r74044 - in pypy/trunk/pypy: lib/app_test objspace/std rlib/test translator/backendopt Message-ID: <20100425015345.658AF282B90@codespeak.net> Author: benjamin Date: Sun Apr 25 03:53:43 2010 New Revision: 74044 Modified: pypy/trunk/pypy/lib/app_test/test_deque_extra.py pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/rlib/test/test_rope.py pypy/trunk/pypy/translator/backendopt/support.py Log: remove compatibility code for reversed and sorted Modified: pypy/trunk/pypy/lib/app_test/test_deque_extra.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/test_deque_extra.py (original) +++ pypy/trunk/pypy/lib/app_test/test_deque_extra.py Sun Apr 25 03:53:43 2010 @@ -1,10 +1,5 @@ # Deque Tests -# for passing the test on top of 2.3 -from py.builtin import reversed -import pypy.lib.collections -pypy.lib.collections.reversed = reversed - n = 10 class Test_deque: Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Sun Apr 25 03:53:43 2010 @@ -512,7 +512,7 @@ self._dict_infos.append(self) def __repr__(self): args = [] - for k in py.builtin.sorted(self.__dict__): + for k in sorted(self.__dict__): v = self.__dict__[k] if v != 0: args.append('%s=%r'%(k, v)) @@ -613,7 +613,7 @@ del DictInfo._dict_infos[-1] tmpl = 'os.write(fd, "%(attr)s" + ": " + str(info.%(attr)s) + "\\n")' bodySrc = [] -for attr in py.builtin.sorted(_example.__dict__): +for attr in sorted(_example.__dict__): if attr == 'sig': continue bodySrc.append(tmpl%locals()) Modified: pypy/trunk/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rope.py (original) +++ pypy/trunk/pypy/rlib/test/test_rope.py Sun Apr 25 03:53:43 2010 @@ -223,7 +223,7 @@ def test_reverse_iteration(): rope, real_st = make_random_string(200) iter = ReverseItemIterator(rope) - for c in py.builtin.reversed(real_st): + for c in reversed(real_st): c2 = iter.nextchar() assert c2 == c py.test.raises(StopIteration, iter.nextchar) @@ -236,12 +236,12 @@ def test_reverse_iteration_unicode(): rope, real_st = make_random_string(200, unicode=True) iter = ReverseItemIterator(rope) - for c in py.builtin.reversed(real_st): + for c in reversed(real_st): c2 = iter.nextunichar() assert c2 == c py.test.raises(StopIteration, iter.nextchar) iter = ReverseItemIterator(rope) - for c in py.builtin.reversed(real_st): + for c in reversed(real_st): c2 = iter.nextint() assert c2 == ord(c) py.test.raises(StopIteration, iter.nextchar) Modified: pypy/trunk/pypy/translator/backendopt/support.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/support.py (original) +++ pypy/trunk/pypy/translator/backendopt/support.py Sun Apr 25 03:53:43 2010 @@ -151,7 +151,8 @@ def compute_reachability(graph): reachable = {} blocks = list(graph.iterblocks()) - for block in py.builtin.reversed(blocks): # this order should make the reuse path more likely + # Reversed order should make the reuse path more likely. + for block in reversed(blocks): reach = set() scheduled = [block] while scheduled: From fijal at codespeak.net Sun Apr 25 06:19:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Apr 2010 06:19:13 +0200 (CEST) Subject: [pypy-svn] r74045 - pypy/branch/rpython-iterator Message-ID: <20100425041913.D94EF282B90@codespeak.net> Author: fijal Date: Sun Apr 25 06:19:12 2010 New Revision: 74045 Removed: pypy/branch/rpython-iterator/ Log: Remove branch - the experiment has failed From fijall at gmail.com Sun Apr 25 06:18:45 2010 From: fijall at gmail.com (Maciej Fijalkowski) Date: Sat, 24 Apr 2010 22:18:45 -0600 Subject: [pypy-svn] r74042 - in pypy/trunk/pypy/annotation: . test In-Reply-To: <20100424214710.82510282B90@codespeak.net> References: <20100424214710.82510282B90@codespeak.net> Message-ID: Hey. Can we use some other exception for such crashes (like NotImplementedError) instead of assertion error? On Sat, Apr 24, 2010 at 3:47 PM, wrote: > Author: benjamin > Date: Sat Apr 24 23:47:08 2010 > New Revision: 74042 > > Modified: > ? pypy/trunk/pypy/annotation/description.py > ? pypy/trunk/pypy/annotation/test/test_annrpython.py > Log: > explode sooner rather than later on class methods > > Modified: pypy/trunk/pypy/annotation/description.py > ============================================================================== > --- pypy/trunk/pypy/annotation/description.py ? (original) > +++ pypy/trunk/pypy/annotation/description.py ? Sat Apr 24 23:47:08 2010 > @@ -595,6 +595,8 @@ > ? ? ? ? ? ? if isinstance(value, staticmethod): ? # special case > ? ? ? ? ? ? ? ? value = value.__get__(42) > ? ? ? ? ? ? ? ? classdef = None ? # don't bind > + ? ? ? ? ? ?elif isinstance(value, classmethod): > + ? ? ? ? ? ? ? ?raise AssertionError("classmethods are not supported") > ? ? ? ? ? ? s_value = self.bookkeeper.immutablevalue(value) > ? ? ? ? ? ? if classdef is not None: > ? ? ? ? ? ? ? ? s_value = s_value.bind_callables_under(classdef, name) > > Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py > ============================================================================== > --- pypy/trunk/pypy/annotation/test/test_annrpython.py ?(original) > +++ pypy/trunk/pypy/annotation/test/test_annrpython.py ?Sat Apr 24 23:47:08 2010 > @@ -234,6 +234,16 @@ > ? ? ? ? s = a.build_types(f, [int]) > ? ? ? ? assert isinstance(s, annmodel.SomeInteger) > > + ? ?def test_classmethod(self): > + ? ? ? ?class X(object): > + ? ? ? ? ? ?@classmethod > + ? ? ? ? ? ?def meth(cls): > + ? ? ? ? ? ? ? ?return None > + ? ? ? ?def f(): > + ? ? ? ? ? ?return X().meth() > + ? ? ? ?a = self.RPythonAnnotator() > + ? ? ? ?py.test.raises(AssertionError, a.build_types, f, ?[]) > + > ? ? def test_methodcall1(self): > ? ? ? ? a = self.RPythonAnnotator() > ? ? ? ? s = a.build_types(snippet._methodcall1, [int]) > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From arigo at codespeak.net Sun Apr 25 10:18:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 10:18:03 +0200 (CEST) Subject: [pypy-svn] r74046 - in pypy/branch/blackhole-improvement/pypy: jit/backend jit/backend/llgraph jit/codewriter jit/codewriter/test jit/metainterp jit/metainterp/test rpython/lltypesystem Message-ID: <20100425081803.0EEEC282C05@codespeak.net> Author: arigo Date: Sun Apr 25 10:18:01 2010 New Revision: 74046 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Log: General slow progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Sun Apr 25 10:18:01 2010 @@ -736,7 +736,7 @@ size = get_class_size(self.memocast, vtable) result = do_new(size) value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable, self.memocast) + value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) return result def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): @@ -785,7 +785,7 @@ err_result = 0.0 else: raise NotImplementedError - return _do_call_common(func, self.memocast, err_result) + return _do_call_common(func, err_result) op_call_pure = op_call @@ -925,7 +925,7 @@ def op_call(self, calldescr, func, *args): sm = ootype.cast_from_object(calldescr.FUNC, func) - newargs = cast_call_args(calldescr.FUNC.ARGS, args, self.memocast) + newargs = cast_call_args(calldescr.FUNC.ARGS, args) res = call_maybe_on_top_of_llinterp(sm, newargs) if isinstance(calldescr.FUNC.RESULT, ootype.OOType): return ootype.cast_to_object(res) @@ -937,7 +937,7 @@ METH = descr.METH obj = ootype.cast_from_object(descr.SELFTYPE, obj) meth = getattr(obj, descr.methname) - newargs = cast_call_args(METH.ARGS, args, self.memocast) + newargs = cast_call_args(METH.ARGS, args) res = call_maybe_on_top_of_llinterp(meth, newargs) if isinstance(METH.RESULT, ootype.OOType): return ootype.cast_to_object(res) @@ -973,27 +973,27 @@ # ____________________________________________________________ -def cast_to_int(x, memocast): +def cast_to_int(x): TP = lltype.typeOf(x) if isinstance(TP, lltype.Ptr): - return cast_adr_to_int(memocast, llmemory.cast_ptr_to_adr(x)) + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) if TP == llmemory.Address: - return cast_adr_to_int(memocast, x) + return llmemory.cast_adr_to_int(x) return lltype.cast_primitive(lltype.Signed, x) -def cast_from_int(TYPE, x, memocast): +def cast_from_int(TYPE, x): if isinstance(TYPE, lltype.Ptr): - if isinstance(x, (int, long)): - x = cast_int_to_adr(memocast, x) + if isinstance(x, (int, long, llmemory.AddressAsInt)): + x = llmemory.cast_int_to_adr(x) return llmemory.cast_adr_to_ptr(x, TYPE) elif TYPE == llmemory.Address: - if isinstance(x, (int, long)): - x = cast_int_to_adr(memocast, x) + if isinstance(x, (int, long, llmemory.AddressAsInt)): + x = llmemory.cast_int_to_adr(x) assert lltype.typeOf(x) == llmemory.Address return x else: if lltype.typeOf(x) == llmemory.Address: - x = cast_adr_to_int(memocast, x) + x = llmemory.cast_adr_to_int(x) return lltype.cast_primitive(TYPE, x) def cast_to_ptr(x): @@ -1161,22 +1161,22 @@ memocast = MemoCast() return _to_opaque(memocast) -def cast_adr_to_int(memocast, adr): - # xxx slow - assert lltype.typeOf(adr) == llmemory.Address - memocast = _from_opaque(memocast) - addresses = memocast.addresses - for i in xrange(len(addresses)-1, -1, -1): - if addresses[i] == adr: - return i - i = len(addresses) - addresses.append(adr) - return i - -def cast_int_to_adr(memocast, int): - memocast = _from_opaque(memocast) - assert 0 <= int < len(memocast.addresses) - return memocast.addresses[int] +##def cast_adr_to_int(memocast, adr): +## # xxx slow +## assert lltype.typeOf(adr) == llmemory.Address +## memocast = _from_opaque(memocast) +## addresses = memocast.addresses +## for i in xrange(len(addresses)-1, -1, -1): +## if addresses[i] == adr: +## return i +## i = len(addresses) +## addresses.append(adr) +## return i + +##def cast_int_to_adr(memocast, int): +## memocast = _from_opaque(memocast) +## assert 0 <= int < len(memocast.addresses) +## return memocast.addresses[int] def get_class_size(memocast, vtable): memocast = _from_opaque(memocast) @@ -1339,25 +1339,29 @@ # ---------- call ---------- -_call_args = [] +_call_args_i = [] +_call_args_r = [] +_call_args_f = [] def do_call_pushint(x): - _call_args.append(x) - -def do_call_pushfloat(x): - _call_args.append(x) + _call_args_i.append(x) def do_call_pushptr(x): - _call_args.append(x) + _call_args_r.append(x) + +def do_call_pushfloat(x): + _call_args_f.append(x) -def _do_call_common(f, memocast, err_result=None): +def _do_call_common(f, err_result=None): global _last_exception assert _last_exception is None, "exception left behind" - ptr = cast_int_to_adr(memocast, f).ptr + ptr = llmemory.cast_int_to_adr(f).ptr FUNC = lltype.typeOf(ptr).TO ARGS = FUNC.ARGS - args = cast_call_args(ARGS, _call_args, memocast) - del _call_args[:] + args = cast_call_args(ARGS, _call_args_i, _call_args_r, _call_args_f) + del _call_args_i[:] + del _call_args_r[:] + del _call_args_f[:] assert len(ARGS) == len(args) try: if hasattr(ptr._obj, 'graph'): @@ -1370,39 +1374,46 @@ result = err_result return result -def do_call_void(f, memocast): - _do_call_common(f, memocast) +def do_call_void(f): + _do_call_common(f) -def do_call_int(f, memocast): - x = _do_call_common(f, memocast, 0) - return cast_to_int(x, memocast) +def do_call_int(f): + x = _do_call_common(f, 0) + return cast_to_int(x) -def do_call_float(f, memocast): - x = _do_call_common(f, memocast, 0) +def do_call_float(f): + x = _do_call_common(f, 0) return cast_to_float(x) -def do_call_ptr(f, memocast): - x = _do_call_common(f, memocast, lltype.nullptr(llmemory.GCREF.TO)) +def do_call_ptr(f): + x = _do_call_common(f, lltype.nullptr(llmemory.GCREF.TO)) return cast_to_ptr(x) -def cast_call_args(ARGS, args, memocast): - argsiter = iter(args) +def cast_call_args(ARGS, args_i, args_r, args_f): + argsiter_i = iter(args_i) + argsiter_r = iter(args_r) + argsiter_f = iter(args_f) args = [] for TYPE in ARGS: if TYPE is lltype.Void: x = None else: - x = argsiter.next() if isinstance(TYPE, ootype.OOType): + x = argsiter_r.next() x = ootype.cast_from_object(TYPE, x) elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': + x = argsiter_r.next() x = cast_from_ptr(TYPE, x) elif TYPE is lltype.Float: + x = argsiter_f.next() x = cast_from_float(TYPE, x) else: - x = cast_from_int(TYPE, x, memocast) + x = argsiter_i.next() + x = cast_from_int(TYPE, x) args.append(x) - assert list(argsiter) == [] + assert list(argsiter_i) == [] + assert list(argsiter_r) == [] + assert list(argsiter_f) == [] return args @@ -1525,8 +1536,8 @@ setannotation(get_frame_forced_token, annmodel.SomeAddress()) setannotation(new_memo_cast, s_MemoCast) -setannotation(cast_adr_to_int, annmodel.SomeInteger()) -setannotation(cast_int_to_adr, annmodel.SomeAddress()) +##setannotation(cast_adr_to_int, annmodel.SomeInteger()) +##setannotation(cast_int_to_adr, annmodel.SomeAddress()) setannotation(set_class_size, annmodel.s_None) setannotation(do_arraylen_gc, annmodel.SomeInteger()) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Sun Apr 25 10:18:01 2010 @@ -496,26 +496,31 @@ newvalue = newvaluebox.getint() llimpl.do_unicodesetitem(0, string, index, newvalue) - def do_call(self, args, calldescr): + def bh_call_i(self, func, calldescr, args_i, args_r, args_f): + self._prepare_call(INT, calldescr, args_i, args_r, args_f) + return llimpl.do_call_int(func) + def bh_call_r(self, func, calldescr, args_i, args_r, args_f): + self._prepare_call(REF, calldescr, args_i, args_r, args_f) + return llimpl.do_call_ptr(func) + def bh_call_f(self, func, calldescr, args_i, args_r, args_f): + self._prepare_call(FLOAT, calldescr, args_i, args_r, args_f) + return llimpl.do_call_float(func) + def bh_call_v(self, func, calldescr, args_i, args_r, args_f): + self._prepare_call('v', calldescr, args_i, args_r, args_f) + llimpl.do_call_void(func) + + def _prepare_call(self, resulttypeinfo, calldescr, args_i, args_r, args_f): assert isinstance(calldescr, Descr) - func = args[0].getint() - for arg in args[1:]: - if arg.type == REF: - llimpl.do_call_pushptr(arg.getref_base()) - elif arg.type == FLOAT: - llimpl.do_call_pushfloat(arg.getfloat()) - else: - llimpl.do_call_pushint(arg.getint()) - if calldescr.typeinfo == REF: - return history.BoxPtr(llimpl.do_call_ptr(func, self.memo_cast)) - elif calldescr.typeinfo == INT: - return history.BoxInt(llimpl.do_call_int(func, self.memo_cast)) - elif calldescr.typeinfo == FLOAT: - return history.BoxFloat(llimpl.do_call_float(func, self.memo_cast)) - elif calldescr.typeinfo == 'v': # void - llimpl.do_call_void(func, self.memo_cast) - else: - raise NotImplementedError + assert calldescr.typeinfo == resulttypeinfo + if args_i is not None: + for x in args_i: + llimpl.do_call_pushint(x) + if args_r is not None: + for x in args_r: + llimpl.do_call_pushptr(x) + if args_f is not None: + for x in args_f: + llimpl.do_call_pushfloat(x) def do_cast_ptr_to_int(self, ptrbox): return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), @@ -629,7 +634,7 @@ assert isinstance(typedescr, TypeDescr) return typedescr.getarraylength(box1) - def do_call(self, args, descr): + def do_call_XXX(self, args, descr): assert isinstance(descr, StaticMethDescr) funcbox = args[0] argboxes = args[1:] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Sun Apr 25 10:18:01 2010 @@ -220,6 +220,16 @@ def do_unicodesetitem(self, stringbox, indexbox, charbox): raise NotImplementedError + # blackhole interface + def bh_call_i(self, func, calldescr, args_i, args_r, args_f): + raise NotImplementedError + def bh_call_r(self, func, calldescr, args_i, args_r, args_f): + raise NotImplementedError + def bh_call_f(self, func, calldescr, args_i, args_r, args_f): + raise NotImplementedError + def bh_call_v(self, func, calldescr, args_i, args_r, args_f): + raise NotImplementedError + def do_call(self, args, calldescr): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Sun Apr 25 10:18:01 2010 @@ -199,7 +199,9 @@ def serialize_op(self, op): args = self.flatten_list(op.args) if op.result is not None: - args.append(self.getcolor(op.result)) + kind = getkind(op.result.concretetype) + if kind != 'void': + args.append(self.getcolor(op.result)) self.emitline(op.opname, *args) def getcolor(self, v): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Sun Apr 25 10:18:01 2010 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.history import AbstractDescr -def format_assembler(ssarepr, dump=True): +def format_assembler(ssarepr): """For testing: format a SSARepr as a multiline string.""" from cStringIO import StringIO seen = {} @@ -14,11 +14,11 @@ if isinstance(x, Register): return '%%%s%d' % (x.kind[0], x.index) # e.g. %i1 or %r2 or %f3 elif isinstance(x, Constant): - return '$' + str(x.value) + return '$%r' % (x.value,) elif isinstance(x, TLabel): return getlabelname(x) elif isinstance(x, ListOfKind): - return '%s[%s]' % (x.kind[0], ', '.join(map(repr, x))) + return '%s[%s]' % (x.kind[0].upper(), ', '.join(map(repr, x))) elif isinstance(x, AbstractDescr): return '%r' % (x,) else: @@ -48,7 +48,4 @@ else: print >> output res = output.getvalue() - if dump: - print res return res - Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Sun Apr 25 10:18:01 2010 @@ -12,25 +12,31 @@ t.transform(graph) +class NoOp(Exception): + pass + + class Transformer(object): def __init__(self, cpu=None): self.cpu = cpu def transform(self, graph): + self.graph = graph for block in graph.iterblocks(): rename = {} newoperations = [] for op in block.operations: - if self.is_noop_operation(op): - rename[op.result] = rename.get(op.args[0], op.args[0]) - else: - for i, v in enumerate(op.args): - if v in rename: - op = SpaceOperation(op.opname, op.args[:], - op.result) - op.args[i] = rename[v] + for i, v in enumerate(op.args): + if v in rename: + op = SpaceOperation(op.opname, op.args[:], + op.result) + op.args[i] = rename[v] + try: newoperations.append(self.rewrite_operation(op)) + except NoOp: + if op.result is not None: + rename[op.result] = rename.get(op.args[0], op.args[0]) block.operations = newoperations self.optimize_goto_if_not(block) @@ -62,18 +68,19 @@ # ---------- - def is_noop_operation(self, op): - return op.opname in self._noop_operations - - _noop_operations = {'same_as': True, - 'cast_int_to_char': True, - 'cast_char_to_int': True} - def rewrite_operation(self, op): try: - return _rewrite_ops[op.opname](self, op) + rewrite = _rewrite_ops[op.opname] except KeyError: return op + else: + return rewrite(self, op) + + def rewrite_op_same_as(self, op): raise NoOp + def rewrite_op_cast_int_to_char(self, op): raise NoOp + def rewrite_op_cast_int_to_unichar(self, op): raise NoOp + def rewrite_op_cast_char_to_int(self, op): raise NoOp + def rewrite_op_cast_unichar_to_int(self, op): raise NoOp def rewrite_op_direct_call(self, op): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' @@ -108,6 +115,19 @@ else: raise AssertionError(kind) lst.append(v) + def rewrite_op_hint(self, op): + hints = op.args[1].value + if hints.get('promote') and op.args[0].concretetype is not lltype.Void: + #self.minimize_variables() + from pypy.rpython.lltypesystem.rstr import STR + assert op.args[0].concretetype != lltype.Ptr(STR) + kind = getkind(op.args[0].concretetype) + return SpaceOperation('%s_guard_value' % kind, + [op.args[0]], op.result) + else: + log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) + raise NoOp + # ____________________________________________________________ _rewrite_ops = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Sun Apr 25 10:18:01 2010 @@ -3,6 +3,7 @@ from pypy.tool.algo.color import DependencyGraph from pypy.tool.algo.unionfind import UnionFind from pypy.jit.metainterp.history import getkind +from pypy.jit.codewriter.flatten import ListOfKind def perform_register_allocation(graph, kind): """Perform register allocation for the Variables of the given 'kind' @@ -30,6 +31,10 @@ for v in op.args: if isinstance(v, Variable): die_at[v] = i + elif isinstance(v, ListOfKind): + for v1 in v: + if isinstance(v1, Variable): + die_at[v1] = i if op.result is not None: die_at[op.result] = i die_at.pop(block.exitswitch, None) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Sun Apr 25 10:18:01 2010 @@ -21,7 +21,7 @@ 'int_return/i': 4} def test_integration(): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter + from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder def f(a, b): while a > 2: b += a @@ -29,8 +29,8 @@ return b cw = CodeWriter() jitcode = cw.transform_func_to_jitcode(f, [5, 6]) - blackholeinterp = BlackholeInterpreter() - blackholeinterp.setup_insns(cw.assembler.insns) + blackholeinterpbuilder = BlackholeInterpBuilder(cw) + blackholeinterp = blackholeinterpbuilder.acquire_interp() blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) blackholeinterp.run(jitcode, 0) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Sun Apr 25 10:18:01 2010 @@ -135,7 +135,7 @@ flattener = GraphFlattener(None, fake_regallocs()) flattener.serialize_op(op) self.assert_format(flattener.ssarepr, """ - residual_call_ir_f $12345, i[%i0, %i1], r[%r0, %r1], %f0 + residual_call_ir_f $12345, I[%i0, %i1], R[%r0, %r1], %f0 """) def test_same_as_removal(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Sun Apr 25 10:18:01 2010 @@ -65,7 +65,7 @@ ] asm = format_assembler(ssarepr) expected = """ - foobar i[%i0, $123, %i1] + foobar I[%i0, $123, %i1] """ assert asm == str(py.code.Source(expected)).strip() + '\n' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Sun Apr 25 10:18:01 2010 @@ -1,8 +1,11 @@ import py from pypy.jit.codewriter import support from pypy.jit.codewriter.regalloc import perform_register_allocation -from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind from pypy.jit.codewriter.format import format_assembler +from pypy.objspace.flow.model import Variable, Constant, SpaceOperation +from pypy.objspace.flow.model import FunctionGraph, Block, Link +from pypy.rpython.lltypesystem import lltype class TestRegAlloc: @@ -69,7 +72,7 @@ L1: int_gt %i0, $0, %i2 goto_if_not L2, %i2 - int_rename i[%i1, %i0], i[%i0, %i1] + int_rename I[%i1, %i0], I[%i0, %i1] goto L1 L2: int_return %i1 @@ -102,7 +105,7 @@ L1: int_gt %i0, $0, %i3 goto_if_not L2, %i3 - int_rename i[%i1, %i2, %i0], i[%i0, %i1, %i2] + int_rename I[%i1, %i2, %i0], I[%i0, %i1, %i2] goto L1 L2: int_return %i1 @@ -123,3 +126,22 @@ L2: int_return %i1 """) + + def test_regalloc_call(self): + v1 = Variable(); v1.concretetype = lltype.Signed + v2 = Variable(); v2.concretetype = lltype.Signed + v3 = Variable(); v3.concretetype = lltype.Signed + v4 = Variable(); v4.concretetype = lltype.Signed + block = Block([v1]) + block.operations = [ + SpaceOperation('int_add', [v1, Constant(1, lltype.Signed)], v2), + SpaceOperation('rescall', [ListOfKind('int', [v1, v2])], v3), + ] + graph = FunctionGraph('f', block, v4) + block.closeblock(Link([v3], graph.returnblock)) + # + self.check_assembler(graph, """ + int_add %i0, $1, %i1 + rescall I[%i0, %i1], %i0 + int_return %i0 + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun Apr 25 10:18:01 2010 @@ -1,7 +1,8 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint +from pypy.rlib.objectmodel import we_are_translated from pypy.tool.sourcetools import func_with_new_name -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop @@ -18,7 +19,7 @@ pass class MissingValue(object): - pass + "NOT_RPYTHON" def signedord(c): value = ord(c) @@ -26,18 +27,16 @@ return value -class BlackholeInterpreter(object): +class BlackholeInterpBuilder(object): - def __init__(self, cpu=None): - self.cpu = cpu - self.registers_i = [MissingValue()] * 256 - self.registers_r = [MissingValue()] * 256 - self.registers_f = [MissingValue()] * 256 + def __init__(self, codewriter): + self.cpu = codewriter.cpu + self.setup_insns(codewriter.assembler.insns) + self.setup_descrs(codewriter.assembler.descrs) + self._freeze_() def _freeze_(self): - self.registers_i = [0] * 256 - self.registers_r = [NULL] * 256 - self.registers_f = [0.0] * 256 + self.blackholeinterps = [] return False def setup_insns(self, insns): @@ -54,22 +53,25 @@ all_funcs.append(self._get_method(name, argcodes)) all_funcs = unrolling_iterable(enumerate(all_funcs)) # - def dispatch_loop(code, position): + def dispatch_loop(self, code, position): while True: opcode = ord(code[position]) position += 1 for i, func in all_funcs: if opcode == i: - position = func(code, position) + position = func(self, code, position) break else: raise AssertionError("bad opcode") dispatch_loop._dont_inline_ = True self.dispatch_loop = dispatch_loop + def setup_descrs(self, descrs): + self.descrs = descrs + def _get_method(self, name, argcodes): # - def handler(code, position): + def handler(self, code, position): args = () next_argcode = 0 for argtype in argtypes: @@ -106,16 +108,37 @@ position += length elif argtype == 'pc': value = position + elif argtype == 'd': + assert argcodes[next_argcode] == 'd' + next_argcode = next_argcode + 1 + index = ord(code[position]) | (ord(code[position+1])<<8) + value = self.descrs[index] + position += 2 else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) - result = boundmethod(*args) + result = unboundmethod(self, *args) if resulttype == 'i': # argcode should be 'i' too assert argcodes[next_argcode] == 'i' next_argcode = next_argcode + 1 + assert lltype.typeOf(result) == lltype.Signed self.registers_i[ord(code[position])] = result position += 1 + elif resulttype == 'r': + # argcode should be 'r' too + assert argcodes[next_argcode] == 'r' + next_argcode = next_argcode + 1 + assert lltype.typeOf(result) == llmemory.GCREF + self.registers_r[ord(code[position])] = result + position += 1 + elif resulttype == 'f': + # argcode should be 'f' too + assert argcodes[next_argcode] == 'f' + next_argcode = next_argcode + 1 + assert lltype.typeOf(result) == lltype.Float + self.registers_f[ord(code[position])] = result + position += 1 elif resulttype == 'L': position = result else: @@ -124,12 +147,40 @@ assert next_argcode == len(argcodes) return position # - boundmethod = getattr(self, 'opimpl_' + name) - argtypes = unrolling_iterable(boundmethod.argtypes) - resulttype = boundmethod.resulttype + unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name) + argtypes = unrolling_iterable(unboundmethod.argtypes) + resulttype = unboundmethod.resulttype handler = func_with_new_name(handler, 'handler_' + name) return handler + def acquire_interp(self): + if len(self.blackholeinterps) > 0: + return self.blackholeinterps.pop() + else: + return BlackholeInterpreter(self) + + def release_interp(self, interp): + self.blackholeinterps.append(interp) + + +class BlackholeInterpreter(object): + + def __init__(self, builder): + self.cpu = builder.cpu + self.dispatch_loop = builder.dispatch_loop + self.descrs = builder.descrs + if we_are_translated(): + default_i = 0 + default_r = lltype.nullptr(llmemory.GCREF.TO) + default_f = 0.0 + else: + default_i = MissingValue() + default_r = MissingValue() + default_f = MissingValue() + self.registers_i = [default_i] * 256 + self.registers_r = [default_r] * 256 + self.registers_f = [default_f] * 256 + def setarg_i(self, index, value): self.registers_i[index] = value @@ -139,7 +190,7 @@ self.copy_constants(self.registers_f, jitcode.constants_f) code = jitcode.code try: - self.dispatch_loop(code, position) + self.dispatch_loop(self, code, position) except LeaveFrame: pass @@ -188,6 +239,10 @@ def opimpl_float_copy(self, a): return a + opimpl_int_guard_value = opimpl_int_copy + opimpl_ref_guard_value = opimpl_ref_copy + opimpl_float_guard_value = opimpl_float_copy + @arguments("L", "i", "i", "pc", returns="L") def opimpl_goto_if_not_int_lt(self, target, a, b, pc): if a < b: @@ -234,24 +289,41 @@ def opimpl_goto(self, target): return target - @arguments("i", "I", "R", returns="i") - def opimpl_residual_call_ir_i(self, function, args_i, args_r): - # XXX! - assert not args_r - return function(*args_i) - - @arguments("i", "I", "R", returns="r") - def opimpl_residual_call_ir_r(self, function, args_i, args_r): - # XXX! - assert not args_r - return function(*args_i) - - @arguments("i", "R", returns="i") - def opimpl_residual_call_r_i(self, function, args_r): - # XXX! - return function(*args_r) - - @arguments("i", "R", returns="r") - def opimpl_residual_call_r_r(self, function, args_r): - # XXX! - return function(*args_r) + @arguments("i", "d", "R", returns="i") + def opimpl_residual_call_r_i(self, func, calldescr, args_r): + return self.cpu.bh_call_i(func, calldescr, None, args_r, None) + @arguments("i", "d", "R", returns="r") + def opimpl_residual_call_r_r(self, func, calldescr, args_r): + return self.cpu.bh_call_r(func, calldescr, None, args_r, None) + @arguments("i", "d", "R", returns="f") + def opimpl_residual_call_r_f(self, func, calldescr, args_r): + return self.cpu.bh_call_f(func, calldescr, None, args_r, None) + @arguments("i", "d", "R", returns="v") + def opimpl_residual_call_r_v(self, func, calldescr, args_r): + return self.cpu.bh_call_v(func, calldescr, None, args_r, None) + + @arguments("i", "d", "I", "R", returns="i") + def opimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): + return self.cpu.bh_call_i(func, calldescr, args_i, args_r, None) + @arguments("i", "d", "I", "R", returns="r") + def opimpl_residual_call_ir_r(self, func, calldescr, args_i, args_r): + return self.cpu.bh_call_r(func, calldescr, args_i, args_r, None) + @arguments("i", "d", "I", "R", returns="f") + def opimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): + return self.cpu.bh_call_f(func, calldescr, args_i, args_r, None) + @arguments("i", "d", "I", "R", returns="v") + def opimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): + return self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) + + @arguments("i", "d", "I", "R", "F", returns="i") + def opimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): + return self.cpu.bh_call_i(func, calldescr, args_i, args_r, args_f) + @arguments("i", "d", "I", "R", "F", returns="r") + def opimpl_residual_call_irf_r(self, func, calldescr,args_i,args_r,args_f): + return self.cpu.bh_call_r(func, calldescr, args_i, args_r, args_f) + @arguments("i", "d", "I", "R", "F", returns="f") + def opimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): + return self.cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) + @arguments("i", "d", "I", "R", "F", returns="v") + def opimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): + return self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Sun Apr 25 10:18:01 2010 @@ -12,21 +12,21 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -def _get_jitcodes(func, values, type_system): +def _get_jitcodes(CPUClass, func, values, type_system): from pypy.jit.codewriter import support, codewriter rtyper = support.annotate(func, values, type_system=type_system) graphs = rtyper.annotator.translator.graphs - cw = codewriter.CodeWriter(rtyper) + stats = history.Stats() + cpu = CPUClass(rtyper, stats, None, False) + cw = codewriter.CodeWriter(cpu) mainjitcode = cw.make_jitcodes(graphs[0], verbose=True) return cw, mainjitcode -def _run_with_blackhole(CPUClass, cw, mainjitcode, args): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - stats = history.Stats() - cpu = CPUClass(cw.rtyper, stats, None, False) - blackholeinterp = BlackholeInterpreter(cpu) - blackholeinterp.setup_insns(cw.assembler.insns) +def _run_with_blackhole(cw, mainjitcode, args): + from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder + blackholeinterpbuilder = BlackholeInterpBuilder(cw) + blackholeinterp = blackholeinterpbuilder.acquire_interp() for i, value in enumerate(args): blackholeinterp.setarg_i(i, value) blackholeinterp.run(mainjitcode, 0) @@ -84,9 +84,10 @@ def interp_operations(self, f, args, **kwds): # get the JitCodes for the function f - cw, mainjitcode = _get_jitcodes(f, args, self.type_system) + cw, mainjitcode = _get_jitcodes(self.CPUClass, f, args, + self.type_system) # try to run it with blackhole.py - result = _run_with_blackhole(self.CPUClass, cw, mainjitcode, args) + result = _run_with_blackhole(cw, mainjitcode, args) # try to run it with pyjitpl.py # -- XXX --- missing return result @@ -148,6 +149,7 @@ def check_operations_history(self, expected=None, **isns): # this can be used after interp_operations + py.test.skip("XXX") self.metainterp.staticdata.stats.check_history(expected, **isns) Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/llmemory.py Sun Apr 25 10:18:01 2010 @@ -503,6 +503,10 @@ return model.SomeInteger() def lltype(self): return lltype.Signed + def __cmp__(self, other): + if isinstance(other, AddressAsInt): + return cmp(self.adr, other.adr) + return Symbolic.__cmp__(self, other) # ____________________________________________________________ From arigo at codespeak.net Sun Apr 25 10:55:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 10:55:43 +0200 (CEST) Subject: [pypy-svn] r74047 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter metainterp Message-ID: <20100425085543.56B46282C05@codespeak.net> Author: arigo Date: Sun Apr 25 10:55:41 2010 New Revision: 74047 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: test_basic.test_format passes. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Sun Apr 25 10:55:41 2010 @@ -422,6 +422,10 @@ count = countbox.getint() return history.BoxPtr(llimpl.do_new_array(size.ofs, count)) + def bh_new_array(self, arraydescr, length): + assert isinstance(arraydescr, Descr) + return llimpl.do_new_array(arraydescr.ofs, length) + def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): assert isinstance(arraydescr, Descr) array = arraybox.getref_base() @@ -439,6 +443,10 @@ else: raise NotImplementedError + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): + assert isinstance(arraydescr, Descr) + llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) + def do_setfield_gc(self, structbox, newvaluebox, fielddescr): assert isinstance(fielddescr, Descr) struct = structbox.getref_base() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Sun Apr 25 10:55:41 2010 @@ -192,13 +192,23 @@ def do_new_with_vtable(self, classbox): raise NotImplementedError - + def do_new_array(self, lengthbox, arraydescr): raise NotImplementedError + + def bh_new_array(self, arraydescr, length): + raise NotImplementedError def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): raise NotImplementedError + def bh_setarrayitem_gc_i(self, arraydescr, array, index, newvalue): + raise NotImplementedError + def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): + raise NotImplementedError + def bh_setarrayitem_gc_f(self, arraydescr, array, index, newvalue): + raise NotImplementedError + def do_setarrayitem_raw(self, arraybox, indexbox, newvaluebox, arraydescr): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Sun Apr 25 10:55:41 2010 @@ -19,14 +19,35 @@ return self.transform_graph_to_jitcode(graph) def transform_graph_to_jitcode(self, graph, verbose=False): + """Transform a graph into a JitCode containing the same bytecode + in a different format. Note that the original 'graph' is mangled + by the process and should not be used any more. + """ + # + # step 1: mangle the graph so that it contains the final instructions + # that we want in the JitCode, but still as a control flow graph transform_graph(graph, self.cpu) + # + # step 2: perform register allocation on it regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) + # + # step 3: flatten the graph to produce human-readable "assembler", + # which means mostly producing a linear list of operations and + # inserting jumps or conditional jumps. This is a list of tuples + # of the shape ("opname", arg1, ..., argN) or (Label(...),). ssarepr = flatten_graph(graph, regallocs) + # + # if 'verbose', print the resulting assembler if verbose: print graph print indent(format_assembler(ssarepr), 4) + # + # step 4: "assemble" it into a JitCode, which contains a sequence + # of bytes and lists of constants. It's during this step that + # constants are cast to their normalized type (Signed, GCREF or + # Float). jitcode = self.assembler.assemble(ssarepr) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Sun Apr 25 10:55:41 2010 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rstr from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation from pypy.jit.codewriter.flatten import ListOfKind @@ -119,8 +119,7 @@ hints = op.args[1].value if hints.get('promote') and op.args[0].concretetype is not lltype.Void: #self.minimize_variables() - from pypy.rpython.lltypesystem.rstr import STR - assert op.args[0].concretetype != lltype.Ptr(STR) + assert op.args[0].concretetype != lltype.Ptr(rstr.STR) kind = getkind(op.args[0].concretetype) return SpaceOperation('%s_guard_value' % kind, [op.args[0]], op.result) @@ -128,6 +127,43 @@ log.WARNING('ignoring hint %r at %r' % (hints, self.graph)) raise NoOp + def rewrite_op_malloc_varsize(self, op): + assert op.args[1].value == {'flavor': 'gc'} + if op.args[0].value == rstr.STR: + return SpaceOperation('newstr', [op.args[2]], op.result) + elif op.args[0].value == rstr.UNICODE: + return SpaceOperation('newunicode', [op.args[2]], op.result) + else: + # XXX only strings or simple arrays for now + ARRAY = op.args[0].value + arraydescr = self.cpu.arraydescrof(ARRAY) + return SpaceOperation('new_array', [arraydescr, op.args[2]], + op.result) + + def rewrite_op_setarrayitem(self, op): + ARRAY = op.args[0].concretetype.TO + assert ARRAY._gckind == 'gc' + if self._array_of_voids(ARRAY): + return +## if op.args[0] in self.vable_array_vars: # for virtualizables +## (v_base, arrayindex) = self.vable_array_vars[op.args[0]] +## self.emit('setarrayitem_vable', +## self.var_position(v_base), +## arrayindex, +## self.var_position(op.args[1]), +## self.var_position(op.args[2])) +## return + arraydescr = self.cpu.arraydescrof(ARRAY) + kind = getkind(op.args[2].concretetype) + return SpaceOperation('setarrayitem_gc_%s' % kind[0], + [arraydescr] + op.args, None) + + def _array_of_voids(self, ARRAY): + #if isinstance(ARRAY, ootype.Array): + # return ARRAY.ITEM == ootype.Void + #else: + return ARRAY.OF == lltype.Void + # ____________________________________________________________ _rewrite_ops = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Sun Apr 25 10:55:41 2010 @@ -62,7 +62,8 @@ except KeyError: pass die_index += 1 - if getkind(op.result.concretetype) == self.kind: + if (op.result is not None and + getkind(op.result.concretetype) == self.kind): dg.add_node(op.result) for v in livevars: if getkind(v.concretetype) == self.kind: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun Apr 25 10:55:41 2010 @@ -75,15 +75,20 @@ args = () next_argcode = 0 for argtype in argtypes: - if argtype == 'i': - # argcode can be 'i' or 'c'; 'c' stands for a single - # signed byte that gives the value of a small constant. + if argtype == 'i' or argtype == 'r' or argtype == 'f': + # if argtype is 'i', then argcode can be 'i' or 'c'; + # 'c' stands for a single signed byte that gives the + # value of a small constant. argcode = argcodes[next_argcode] next_argcode = next_argcode + 1 if argcode == 'i': value = self.registers_i[ord(code[position])] elif argcode == 'c': value = signedord(code[position]) + elif argcode == 'r': + value = self.registers_r[ord(code[position])] + elif argcode == 'f': + value = self.registers_f[ord(code[position])] else: raise AssertionError("bad argcode") position += 1 @@ -289,6 +294,9 @@ def opimpl_goto(self, target): return target + # ---------- + # the following operations are directly implemented by the backend + @arguments("i", "d", "R", returns="i") def opimpl_residual_call_r_i(self, func, calldescr, args_r): return self.cpu.bh_call_i(func, calldescr, None, args_r, None) @@ -327,3 +335,10 @@ @arguments("i", "d", "I", "R", "F", returns="v") def opimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) + + @arguments("d", "i", returns="r") + def opimpl_new_array(self, arraydescr, length): + return self.cpu.bh_new_array(arraydescr, length) + @arguments("d", "r", "i", "r") + def opimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): + self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) From arigo at codespeak.net Sun Apr 25 11:19:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 11:19:55 +0200 (CEST) Subject: [pypy-svn] r74048 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100425091955.1AC2C282BD8@codespeak.net> Author: arigo Date: Sun Apr 25 11:19:53 2010 New Revision: 74048 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Not-too-big switches. Integer only for now. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Sun Apr 25 11:19:53 2010 @@ -129,8 +129,9 @@ link = block.exits[0] assert link.exitcase is None self.make_link(link) - else: - assert len(block.exits) == 2, "XXX" + elif len(block.exits) == 2 and ( + isinstance(block.exitswitch, tuple) or + block.exitswitch.concretetype == lltype.Bool): linkfalse, linktrue = block.exits if linkfalse.llexitcase == True: linkfalse, linktrue = linktrue, linkfalse @@ -150,6 +151,31 @@ # false path: self.emitline(Label(linkfalse)) self.make_link(linkfalse) + else: + switches = [link for link in block.exits + if link.exitcase != 'default'] + if len(switches) >= 5 and isinstance(block.exitswitch.concretetype, + lltype.Primitive): + XXX + else: + kind = getkind(block.exitswitch.concretetype) + assert kind == 'int' # XXX + for switch in switches: + # make the case described by 'switch' + self.emitline('goto_if_not_int_eq', + TLabel(switch), + self.getcolor(block.exitswitch), + Constant(switch.llexitcase, + block.exitswitch.concretetype)) + # emit code for the "taken" path + self.make_link(switch) + # finally, emit the label for the "non-taken" path + self.emitline(Label(switch)) + # + if block.exits[-1].exitcase == 'default': + self.make_link(block.exits[-1]) + else: + self.emitline('unreachable') def insert_renamings(self, link): renamings = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Sun Apr 25 11:19:53 2010 @@ -157,3 +157,22 @@ self.assert_format(flattener.ssarepr, """ foobar hi_there! """) + + def test_switch(self): + def f(n): + if n == -5: return 12 + elif n == 2: return 51 + elif n == 7: return 1212 + else: return 42 + self.encoding_test(f, [65], """ + goto_if_not_int_eq L1, %i0, $-5 + int_return $12 + L1: + goto_if_not_int_eq L2, %i0, $2 + int_return $51 + L2: + goto_if_not_int_eq L3, %i0, $7 + int_return $1212 + L3: + int_return $42 + """) From arigo at codespeak.net Sun Apr 25 11:38:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 11:38:54 +0200 (CEST) Subject: [pypy-svn] r74049 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100425093854.7AC4A282BD8@codespeak.net> Author: arigo Date: Sun Apr 25 11:38:53 2010 New Revision: 74049 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Add SwitchDictDescr to handle large switches. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Sun Apr 25 11:38:53 2010 @@ -42,6 +42,9 @@ def __iter__(self): return iter(self.content) +class SwitchDictDescr(AbstractDescr): + "Get a 'dict' attribute mapping integer values to bytecode positions." + KINDS = ['int', 'ref', 'float'] # ____________________________________________________________ @@ -126,12 +129,15 @@ def insert_exits(self, block): if len(block.exits) == 1: + # A single link, fall-through link = block.exits[0] assert link.exitcase is None self.make_link(link) + # elif len(block.exits) == 2 and ( isinstance(block.exitswitch, tuple) or block.exitswitch.concretetype == lltype.Bool): + # Two exit links with a boolean condition linkfalse, linktrue = block.exits if linkfalse.llexitcase == True: linkfalse, linktrue = linktrue, linkfalse @@ -151,14 +157,40 @@ # false path: self.emitline(Label(linkfalse)) self.make_link(linkfalse) + # else: + # A switch. + # + def emitdefaultpath(): + if block.exits[-1].exitcase == 'default': + self.make_link(block.exits[-1]) + else: + self.emitline('unreachable') + # switches = [link for link in block.exits if link.exitcase != 'default'] - if len(switches) >= 5 and isinstance(block.exitswitch.concretetype, - lltype.Primitive): - XXX + switches.sort(key=lambda link: link.llexitcase) + kind = getkind(block.exitswitch.concretetype) + if len(switches) >= 5 and kind == 'int': + # A large switch on an integer, implementable efficiently + # with the help of a SwitchDictDescr + switchdict = SwitchDictDescr() + switchdict._labels = [] + self.emitline('switch', self.getcolor(block.exitswitch), + switchdict) + emitdefaultpath() + # + for switch in switches: + key = lltype.cast_primitive(lltype.Signed, + switch.llexitcase) + switchdict._labels.append((key, TLabel(switch))) + # emit code for that path + self.emitline(Label(switch)) + self.make_link(switch) + # else: - kind = getkind(block.exitswitch.concretetype) + # A switch with several possible answers, though not too + # many of them -- a chain of int_eq comparisons is fine assert kind == 'int' # XXX for switch in switches: # make the case described by 'switch' @@ -172,10 +204,7 @@ # finally, emit the label for the "non-taken" path self.emitline(Label(switch)) # - if block.exits[-1].exitcase == 'default': - self.make_link(block.exits[-1]) - else: - self.emitline('unreachable') + emitdefaultpath() def insert_renamings(self, link): renamings = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Sun Apr 25 11:38:53 2010 @@ -1,7 +1,7 @@ import py from pypy.objspace.flow.model import Constant from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr from pypy.jit.metainterp.history import AbstractDescr @@ -19,19 +19,19 @@ return getlabelname(x) elif isinstance(x, ListOfKind): return '%s[%s]' % (x.kind[0].upper(), ', '.join(map(repr, x))) + elif isinstance(x, SwitchDictDescr): + return '' % ( + ', '.join(['%s:%s' % (key, getlabelname(lbl)) + for key, lbl in x._labels])) elif isinstance(x, AbstractDescr): return '%r' % (x,) else: return '' % (x,) # seenlabels = {} - for asm in ssarepr.insns: - for x in asm: - if isinstance(x, TLabel): - seenlabels[x.name] = -1 labelcount = [0] def getlabelname(lbl): - if seenlabels[lbl.name] == -1: + if lbl.name not in seenlabels: labelcount[0] += 1 seenlabels[lbl.name] = labelcount[0] return 'L%d' % seenlabels[lbl.name] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Sun Apr 25 11:38:53 2010 @@ -176,3 +176,29 @@ L3: int_return $42 """) + + def test_switch_dict(self): + def f(x): + if x == 1: return 61 + elif x == 2: return 511 + elif x == 3: return -22 + elif x == 4: return 81 + elif x == 5: return 17 + elif x == 6: return 54 + return -1 + self.encoding_test(f, [65], """ + switch %i0, + int_return $-1 + L1: + int_return $61 + L2: + int_return $511 + L3: + int_return $-22 + L4: + int_return $81 + L5: + int_return $17 + L6: + int_return $54 + """) From arigo at codespeak.net Sun Apr 25 11:48:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 11:48:53 +0200 (CEST) Subject: [pypy-svn] r74050 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100425094853.97CCF282BD8@codespeak.net> Author: arigo Date: Sun Apr 25 11:48:52 2010 New Revision: 74050 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Log: Oups. Fixes r74049. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Sun Apr 25 11:48:52 2010 @@ -29,9 +29,16 @@ return '' % (x,) # seenlabels = {} + for asm in ssarepr.insns: + for x in asm: + if isinstance(x, TLabel): + seenlabels[x.name] = -1 + elif isinstance(x, SwitchDictDescr): + for _, switch in x._labels: + seenlabels[switch.name] = -1 labelcount = [0] def getlabelname(lbl): - if lbl.name not in seenlabels: + if seenlabels[lbl.name] == -1: labelcount[0] += 1 seenlabels[lbl.name] = labelcount[0] return 'L%d' % seenlabels[lbl.name] From arigo at codespeak.net Sun Apr 25 11:50:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 11:50:11 +0200 (CEST) Subject: [pypy-svn] r74051 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100425095011.76022282BD8@codespeak.net> Author: arigo Date: Sun Apr 25 11:50:09 2010 New Revision: 74051 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Passes test_switch_dict in test_basic. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Sun Apr 25 11:50:09 2010 @@ -1,6 +1,6 @@ from pypy.jit.metainterp.history import AbstractValue, AbstractDescr, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS -from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory @@ -49,6 +49,7 @@ self.constants_f = [] self.label_positions = {} self.tlabel_positions = [] + self.switchdictdescrs = [] self.highest_regs = dict.fromkeys(KINDS, 0) def emit_reg(self, reg): @@ -133,6 +134,8 @@ if x not in self._descr_dict: self._descr_dict[x] = len(self.descrs) self.descrs.append(x) + if isinstance(x, SwitchDictDescr): + self.switchdictdescrs.append(x) num = self._descr_dict[x] assert 0 <= num <= 0xFFFF, "too many AbstractDescrs!" self.code.append(chr(num & 0xFF)) @@ -153,6 +156,11 @@ assert 0 <= target <= 0xFFFF self.code[pos ] = chr(target & 0xFF) self.code[pos+1] = chr(target >> 8) + for descr in self.switchdictdescrs: + descr.dict = {} + for key, switchlabel in descr._labels: + target = self.label_positions[switchlabel.name] + descr.dict[key] = target def check_result(self): # Limitation of the number of registers, from the single-byte encoding Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Sun Apr 25 11:50:09 2010 @@ -4,6 +4,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop +from pypy.jit.codewriter.flatten import SwitchDictDescr def arguments(*argtypes, **kwds): @@ -294,6 +295,14 @@ def opimpl_goto(self, target): return target + @arguments("i", "d", "pc", returns="L") + def opimpl_switch(self, switchvalue, switchdict, pc): + assert isinstance(switchdict, SwitchDictDescr) + try: + return switchdict.dict[switchvalue] + except KeyError: + return pc + # ---------- # the following operations are directly implemented by the backend From arigo at codespeak.net Sun Apr 25 12:06:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Apr 2010 12:06:05 +0200 (CEST) Subject: [pypy-svn] r74052 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20100425100605.76DEF282BD8@codespeak.net> Author: arigo Date: Sun Apr 25 12:06:03 2010 New Revision: 74052 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py Log: Be more permissive here. Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_socket.py Sun Apr 25 12:06:03 2010 @@ -323,7 +323,7 @@ # self.assertRaises(OverflowError, func, 1<= 2.6: - self.assertRaises(OverflowError, func, -1) + self.assertRaises((OverflowError, ValueError), func, -1) def testGetServBy(self): eq = self.assertEqual From afa at codespeak.net Mon Apr 26 10:59:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 10:59:55 +0200 (CEST) Subject: [pypy-svn] r74060 - pypy/trunk/pypy/module/_locale/test Message-ID: <20100426085955.699C5282B90@codespeak.net> Author: afa Date: Mon Apr 26 10:59:52 2010 New Revision: 74060 Modified: pypy/trunk/pypy/module/_locale/test/test_locale.py Log: These _locale tests need the unicode database Modified: pypy/trunk/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/test/test_locale.py (original) +++ pypy/trunk/pypy/module/_locale/test/test_locale.py Mon Apr 26 10:59:52 2010 @@ -5,7 +5,8 @@ class AppTestLocaleTrivia: def setup_class(cls): - cls.space = space = gettestobjspace(usemodules=['_locale']) + cls.space = space = gettestobjspace(usemodules=['_locale', + 'unicodedata']) if sys.platform != 'win32': cls.w_language_en = cls.space.wrap("C") cls.w_language_utf8 = cls.space.wrap("en_US.utf8") From afa at codespeak.net Mon Apr 26 12:47:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 12:47:23 +0200 (CEST) Subject: [pypy-svn] r74061 - in pypy/trunk/pypy: module/_locale rlib Message-ID: <20100426104723.102F2282B90@codespeak.net> Author: afa Date: Mon Apr 26 12:47:21 2010 New Revision: 74061 Modified: pypy/trunk/pypy/module/_locale/interp_locale.py pypy/trunk/pypy/rlib/rlocale.py Log: Move most of _locale.getdefaultlocale to rlib.rlocale. Modified: pypy/trunk/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/trunk/pypy/module/_locale/interp_locale.py (original) +++ pypy/trunk/pypy/module/_locale/interp_locale.py Mon Apr 26 12:47:21 2010 @@ -298,49 +298,7 @@ bind_textdomain_codeset.unwrap_spec = [ObjSpace, str, W_Root] -#___________________________________________________________________ -# getdefaultlocale() implementation for Windows and MacOSX - if sys.platform == 'win32': - from pypy.rlib import rwin32 - LCID = LCTYPE = rwin32.DWORD - GetACP = rlocale.external('GetACP', - [], rffi.INT, - calling_conv='win') - GetLocaleInfo = rlocale.external('GetLocaleInfoA', - [LCID, LCTYPE, rwin32.LPSTR, rffi.INT], rffi.INT, - calling_conv='win') - def getdefaultlocale(space): - encoding = "cp%d" % GetACP() - - BUFSIZE = 50 - buf_lang = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') - buf_country = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') - - try: - if (GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, - rlocale.cConfig.LOCALE_SISO639LANGNAME, - buf_lang, BUFSIZE) and - GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, - rlocale.cConfig.LOCALE_SISO3166CTRYNAME, - buf_country, BUFSIZE)): - lang = rffi.charp2str(buf_lang) - country = rffi.charp2str(buf_country) - return space.newtuple([space.wrap("%s_%s" % (lang, country)), - space.wrap(encoding)]) - - # If we end up here, this windows version didn't know about - # ISO639/ISO3166 names (it's probably Windows 95). Return the - # Windows language identifier instead (a hexadecimal number) - elif GetLocaleInfo(rlocale.cConfig.LOCALE_USER_DEFAULT, - rlocale.cConfig.LOCALE_IDEFAULTLANGUAGE, - buf_lang, BUFSIZE): - lang = rffi.charp2str(buf_lang) - return space.newtuple([space.wrap("0x%s" % lang), - space.wrap(encoding)]) - else: - return space.newtuple([space.w_None, space.wrap(encoding)]) - finally: - lltype.free(buf_lang, flavor='raw') - lltype.free(buf_country, flavor='raw') + language, encoding = rlocale.getdefaultlocale() + return space.newtuple([space.wrap(language), space.wrap(encoding)]) Modified: pypy/trunk/pypy/rlib/rlocale.py ============================================================================== --- pypy/trunk/pypy/rlib/rlocale.py (original) +++ pypy/trunk/pypy/rlib/rlocale.py Mon Apr 26 12:47:21 2010 @@ -172,3 +172,51 @@ if key in constants.values(): return rffi.charp2str(_nl_langinfo(rffi.cast(rffi.INT, key))) raise ValueError + +#___________________________________________________________________ +# getdefaultlocale() implementation for Windows + +if sys.platform == 'win32': + from pypy.rlib import rwin32 + LCID = LCTYPE = rwin32.DWORD + GetACP = external('GetACP', + [], rffi.INT, + calling_conv='win') + GetLocaleInfo = external('GetLocaleInfoA', + [LCID, LCTYPE, rwin32.LPSTR, rffi.INT], rffi.INT, + calling_conv='win') + + def getdefaultlocale(): + encoding = "cp%d" % GetACP() + + BUFSIZE = 50 + buf_lang = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') + buf_country = lltype.malloc(rffi.CCHARP.TO, BUFSIZE, flavor='raw') + + try: + if (GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_SISO639LANGNAME, + buf_lang, BUFSIZE) and + GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_SISO3166CTRYNAME, + buf_country, BUFSIZE)): + lang = rffi.charp2str(buf_lang) + country = rffi.charp2str(buf_country) + language = "%s_%s" % (lang, country) + + # If we end up here, this windows version didn't know about + # ISO639/ISO3166 names (it's probably Windows 95). Return the + # Windows language identifier instead (a hexadecimal number) + elif GetLocaleInfo(cConfig.LOCALE_USER_DEFAULT, + cConfig.LOCALE_IDEFAULTLANGUAGE, + buf_lang, BUFSIZE): + lang = rffi.charp2str(buf_lang) + language = "0x%s" % (lang,) + + else: + language = None + finally: + lltype.free(buf_lang, flavor='raw') + lltype.free(buf_country, flavor='raw') + + return language, encoding From afa at codespeak.net Mon Apr 26 12:52:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 12:52:19 +0200 (CEST) Subject: [pypy-svn] r74062 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426105219.6C91C282B90@codespeak.net> Author: afa Date: Mon Apr 26 12:52:17 2010 New Revision: 74062 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Unused import Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Mon Apr 26 12:52:17 2010 @@ -1,6 +1,5 @@ import os import sys -from weakref import ref from pypy.rpython.lltypesystem import rffi, lltype from pypy.tool.pairtype import extendabletype From afa at codespeak.net Mon Apr 26 12:57:16 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 12:57:16 +0200 (CEST) Subject: [pypy-svn] r74063 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426105716.E93E7282B90@codespeak.net> Author: afa Date: Mon Apr 26 12:57:15 2010 New Revision: 74063 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_SetAttrString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Mon Apr 26 12:57:15 2010 @@ -101,6 +101,12 @@ operation.setattr(space, w_obj, w_name, w_value) return 0 + at cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1) +def PyObject_SetAttrString(space, w_obj, name_ptr, w_value): + w_name = space.wrap(rffi.charp2str(name_ptr)) + operation.setattr(space, w_obj, w_name, w_value) + return 0 + @cpython_api([PyObject], lltype.Void) def PyObject_ClearWeakRefs(space, w_object): w_object.clear_all_weakrefs() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Mon Apr 26 12:57:15 2010 @@ -48,7 +48,12 @@ assert api.PyObject_HasAttr(space.wrap(x), space.wrap('test')) api.PyObject_SetAttr(space.wrap(x), space.wrap('test'), space.wrap(10)) assert x.test == 10 - + + buf = rffi.str2charp('test') + api.PyObject_SetAttrString(space.wrap(x), buf, space.wrap(20)) + rffi.free_charp(buf) + assert x.test == 20 + def test_getattr_string(self, space, api): charp1 = rffi.str2charp("__len__") charp2 = rffi.str2charp("not_real") From afa at codespeak.net Mon Apr 26 13:23:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 13:23:25 +0200 (CEST) Subject: [pypy-svn] r74064 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100426112325.AE3F2282B90@codespeak.net> Author: afa Date: Mon Apr 26 13:23:23 2010 New Revision: 74064 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: Expose the PyCObject structure to C code. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 13:23:23 2010 @@ -49,7 +49,8 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( include_dirs=include_dirs, - includes=['Python.h', 'stdarg.h'] + includes=['Python.h', 'stdarg.h'], + compile_extra=['-DPy_BUILD_CORE'], ) class CConfig_constants: @@ -646,12 +647,14 @@ kwds = {} export_symbols_eci = export_symbols[:] + compile_extra=['-DPy_BUILD_CORE'] + if building_bridge: if sys.platform == "win32": # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] + compile_extra.append("/we4013") else: - kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] + compile_extra.append("-Werror=implicit-function-declaration") export_symbols_eci.append('pypyAPI') else: kwds["includes"] = ['Python.h'] # this is our Python.h @@ -670,6 +673,7 @@ ], separate_module_sources = [code], export_symbols=export_symbols_eci, + compile_extra=compile_extra, **kwds ) return eci Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h Mon Apr 26 13:23:23 2010 @@ -1,5 +1,19 @@ +#ifndef Py_COBJECT_H +#define Py_COBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#if (PY_VERSION_HEX >= 0x02060000 || defined(Py_BUILD_CORE)) typedef struct { PyObject_HEAD + void *cobject; + void *desc; void (*destructor)(void *); } PyCObject; - +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_COBJECT_H */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Mon Apr 26 13:23:23 2010 @@ -8,7 +8,12 @@ destructor_short = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) destructor_long = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real, rffi.VOIDP_real], lltype.Void)) -PyCObjectStruct = cpython_struct('PyCObject', PyObjectFields + (("destructor", destructor_short), )) +PyCObjectStruct = cpython_struct('PyCObject', + PyObjectFields + ( + ("cobject", rffi.VOIDP_real), + ("desc", rffi.VOIDP_real), + ("destructor", destructor_short), + )) PyCObject = lltype.Ptr(PyCObjectStruct) @@ -27,35 +32,31 @@ basestruct=PyCObjectStruct) class W_PyCObjectFromVoidPtr(W_PyCObject): - def __init__(self, space, voidp, desc): - W_PyCObject.__init__(self, space) - self.voidp = voidp - self.pyo = lltype.nullptr(PyCObject.TO) - self.desc = desc - self.space = space + pyo = lltype.nullptr(PyCObjectStruct) def set_pycobject(self, pyo): self.pyo = pyo def __del__(self): - if self.pyo and self.pyo.c_destructor: - if self.desc: + pyo = self.pyo + if pyo and pyo.c_destructor: + if pyo.c_desc: generic_cpy_call(self.space, rffi.cast(destructor_long, - self.pyo.c_destructor), self.voidp, self.desc) + pyo.c_destructor), pyo.c_cobject, pyo.c_desc) else: - generic_cpy_call(self.space, self.pyo.c_destructor, self.voidp) + generic_cpy_call(self.space, pyo.c_destructor, pyo.c_cobject) @cpython_api([rffi.VOIDP_real, destructor_short], PyObject) def PyCObject_FromVoidPtr(space, cobj, destr): """Create a PyCObject from the void * cobj. The destr function will be called when the object is reclaimed, unless it is NULL.""" - w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj, - lltype.nullptr(rffi.VOIDP_real.TO))) + w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space)) assert isinstance(w_pycobject, W_PyCObjectFromVoidPtr) pyo = make_ref(space, w_pycobject) pycobject = rffi.cast(PyCObject, pyo) w_pycobject.set_pycobject(pycobject) + pycobject.c_cobject = cobj pycobject.c_destructor = destr return pyo @@ -64,12 +65,13 @@ """Create a PyCObject from the void * cobj. The destr function will be called when the object is reclaimed. The desc argument can be used to pass extra callback data for the destructor function.""" - w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space, cobj, - desc)) + w_pycobject = space.wrap(W_PyCObjectFromVoidPtr(space)) assert isinstance(w_pycobject, W_PyCObjectFromVoidPtr) pyo = make_ref(space, w_pycobject) pycobject = rffi.cast(PyCObject, pyo) w_pycobject.set_pycobject(pycobject) + pycobject.c_cobject = cobj + pybobject.c_desc = desc pycobject.c_destructor = rffi.cast(destructor_short, destr) return pyo Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Mon Apr 26 13:23:23 2010 @@ -2,9 +2,13 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.pycobject import destructor_short +from pypy.module.cpyext.pycobject import destructor_short, PyCObject class TestPyCObject(BaseApiTest): def test_pycobject(self, space, api): - obj = api.PyCObject_FromVoidPtr(rffi.cast(rffi.VOIDP_real, 0), lltype.nullptr(destructor_short.TO)) - api.Py_DecRef(obj) + ptr = rffi.cast(rffi.VOIDP_real, 1234) + obj = api.PyCObject_FromVoidPtr(ptr, lltype.nullptr(destructor_short.TO)) + try: + assert rffi.cast(PyCObject, obj).c_cobject == ptr + finally: + api.Py_DecRef(obj) From afa at codespeak.net Mon Apr 26 13:46:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 13:46:28 +0200 (CEST) Subject: [pypy-svn] r74067 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426114628.889DC282B90@codespeak.net> Author: afa Date: Mon Apr 26 13:46:26 2010 New Revision: 74067 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: PyCObject_Check, PyCObject_AsVoidPtr Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 13:46:26 2010 @@ -346,21 +346,29 @@ if name in TYPES: TYPES[name].become(TYPE) -def build_type_checkers(type_name, on_space=None): - if on_space is None: - on_space = "w_" + type_name.lower() +def build_type_checkers(type_name, cls=None): + if cls is None: + attrname = "w_" + type_name.lower() + def get_w_type(space): + return getattr(space, attrname) + elif isinstance(cls, str): + def get_w_type(space): + return getattr(space, cls) + else: + def get_w_type(space): + return space.gettypeobject(cls.typedef) check_name = "Py" + type_name + "_Check" @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, name=check_name) def check(space, w_obj): w_obj_type = space.type(w_obj) - w_type = getattr(space, on_space) + w_type = get_w_type(space) return int(space.is_w(w_obj_type, w_type) or space.is_true(space.issubtype(w_obj_type, w_type))) @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL, name=check_name + "Exact") def check_exact(space, w_obj): w_obj_type = space.type(w_obj) - w_type = getattr(space, on_space) + w_type = get_w_type(space) return int(space.is_w(w_obj_type, w_type)) return check, check_exact Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Mon Apr 26 13:46:26 2010 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import make_ref, make_typedescr from pypy.module.cpyext.api import generic_cpy_call, cpython_api, bootstrap_function, \ - PyObject, cpython_struct, PyObjectFields + PyObject, cpython_struct, PyObjectFields, build_type_checkers destructor_short = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) @@ -26,6 +26,8 @@ ) W_PyCObject.typedef.acceptable_as_base_class = False +PyCObject_Check, _ = build_type_checkers("CObject", W_PyCObject) + @bootstrap_function def init_pycobject(space): make_typedescr(W_PyCObject.typedef, @@ -75,4 +77,9 @@ pycobject.c_destructor = rffi.cast(destructor_short, destr) return pyo - + at cpython_api([PyObject], rffi.VOIDP_real, error=lltype.nullptr(rffi.VOIDP_real.TO)) +def PyCObject_AsVoidPtr(space, w_obj): + """Return the object void * that the PyCObject self was + created with.""" + assert isinstance(w_obj, W_PyCObjectFromVoidPtr) + return w_obj.pyo.c_cobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 13:46:26 2010 @@ -840,30 +840,6 @@ instance.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyCObject_Check(space, p): - """Return true if its argument is a PyCObject.""" - raise NotImplementedError - - at cpython_api([{void*}, {void (*destr)(void*}], PyObject) -def PyCObject_FromVoidPtr(space, cobj, )): - """Create a PyCObject from the void * cobj. The destr function - will be called when the object is reclaimed, unless it is NULL.""" - raise NotImplementedError - - at cpython_api([{void*}, {void*}, {void (*destr)(void*}, {void*}], PyObject) -def PyCObject_FromVoidPtrAndDesc(space, cobj, desc, , )): - """Create a PyCObject from the void * cobj. The destr - function will be called when the object is reclaimed. The desc argument can - be used to pass extra callback data for the destructor function.""" - raise NotImplementedError - - at cpython_api([PyObject], {void*}) -def PyCObject_AsVoidPtr(space, self): - """Return the object void * that the PyCObject self was - created with.""" - raise NotImplementedError - @cpython_api([PyObject], {void*}) def PyCObject_GetDesc(space, self): """Return the description void * that the PyCObject self was @@ -3851,13 +3827,6 @@ expression o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real) -def PyObject_SetAttrString(space, o, attr_name, v): - """Set the value of the attribute named attr_name, for object o, to the value - v. Returns -1 on failure. This is the equivalent of the Python statement - o.attr_name = v.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], rffi.INT_real) def PyObject_DelAttr(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Mon Apr 26 13:46:26 2010 @@ -8,7 +8,7 @@ def test_pycobject(self, space, api): ptr = rffi.cast(rffi.VOIDP_real, 1234) obj = api.PyCObject_FromVoidPtr(ptr, lltype.nullptr(destructor_short.TO)) - try: - assert rffi.cast(PyCObject, obj).c_cobject == ptr - finally: - api.Py_DecRef(obj) + assert api.PyCObject_Check(obj) + assert api.PyCObject_AsVoidPtr(obj) == ptr + assert rffi.cast(PyCObject, obj).c_cobject == ptr + api.Py_DecRef(obj) From afa at codespeak.net Mon Apr 26 13:54:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 13:54:19 +0200 (CEST) Subject: [pypy-svn] r74068 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426115419.EFEDC282B90@codespeak.net> Author: afa Date: Mon Apr 26 13:54:18 2010 New Revision: 74068 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: Add a test, and fix a typo. Thanks aleale! Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Mon Apr 26 13:54:18 2010 @@ -73,7 +73,7 @@ pycobject = rffi.cast(PyCObject, pyo) w_pycobject.set_pycobject(pycobject) pycobject.c_cobject = cobj - pybobject.c_desc = desc + pycobject.c_desc = desc pycobject.c_destructor = rffi.cast(destructor_short, destr) return pyo Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Mon Apr 26 13:54:18 2010 @@ -12,3 +12,7 @@ assert api.PyCObject_AsVoidPtr(obj) == ptr assert rffi.cast(PyCObject, obj).c_cobject == ptr api.Py_DecRef(obj) + + obj = api.PyCObject_FromVoidPtrAndDesc(ptr, ptr, + lltype.nullptr(destructor_short.TO)) + api.Py_DecRef(obj) From afa at codespeak.net Mon Apr 26 13:57:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 13:57:54 +0200 (CEST) Subject: [pypy-svn] r74069 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include Message-ID: <20100426115754.425F4282B90@codespeak.net> Author: afa Date: Mon Apr 26 13:57:52 2010 New Revision: 74069 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Log: define the PyAPI_FUNC() macro, and use it to properly declare functions as "dllexport" on Windows. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 13:57:52 2010 @@ -618,13 +618,12 @@ arg = arg.replace('@', 'arg%d' % (i,)).strip() args.append(arg) args = ', '.join(args) or "void" - header = "%s %s(%s)" % (restype, name, args) - pypy_decls.append(header + ";") + pypy_decls.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args)) if api_struct: callargs = ', '.join('arg%d' % (i,) for i in range(len(func.argtypes))) body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) - functions.append('%s\n%s\n' % (header, body)) + functions.append('%s %s(%s)\n%s' % (restype, name, args, body)) for name in VA_TP_LIST: name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Mon Apr 26 13:57:52 2010 @@ -9,6 +9,7 @@ # include # include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) +# define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE #else # define MS_WIN32 1 @@ -16,8 +17,10 @@ # include # define Py_DEPRECATED(VERSION_UNUSED) # ifdef Py_BUILD_CORE +# define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE # define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE # else +# define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE # define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE # endif #endif From afa at codespeak.net Mon Apr 26 14:17:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 14:17:41 +0200 (CEST) Subject: [pypy-svn] r74070 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426121741.29C23282B90@codespeak.net> Author: afa Date: Mon Apr 26 14:17:39 2010 New Revision: 74070 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Now that functions are properly marked with PyAPI_FUNC, there is no need to "/export" them again. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 14:17:39 2010 @@ -486,7 +486,7 @@ def build_bridge(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) + export_symbols = SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -689,7 +689,7 @@ def setup_library(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) + export_symbols = SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() From afa at codespeak.net Mon Apr 26 14:34:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 14:34:20 +0200 (CEST) Subject: [pypy-svn] r74071 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426123420.1F227282B9C@codespeak.net> Author: afa Date: Mon Apr 26 14:34:18 2010 New Revision: 74071 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: PyList_Insert Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Mon Apr 26 14:34:18 2010 @@ -62,6 +62,14 @@ w_list.append(w_item) return 0 + at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) +def PyList_Insert(space, w_list, index, w_item): + """Insert the item item into list list in front of index index. Return + 0 if successful; return -1 and set an exception if unsuccessful. + Analogous to list.insert(index, item).""" + space.call_method(w_list, "insert", space.wrap(index), w_item) + return 0 + @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PyList_GET_SIZE(space, w_list): """Macro form of PyList_Size() without error checking. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Mon Apr 26 14:34:18 2010 @@ -32,6 +32,16 @@ assert api.PyList_Size(space.w_None) == -1 assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + + def test_insert(self, space, api): + w_l = space.newlist([space.w_None, space.w_None]) + assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0 + assert api.PyList_Size(w_l) == 3 + assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0 + assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2 + # insert at index -1: next-to-last + assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0 + assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3 def test_sort(self, space, api): l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)]) From afa at codespeak.net Mon Apr 26 14:39:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 14:39:21 +0200 (CEST) Subject: [pypy-svn] r74072 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426123921.273E3282B90@codespeak.net> Author: afa Date: Mon Apr 26 14:39:19 2010 New Revision: 74072 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Log: PyMethod_Check, PyCFunction_Check Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Mon Apr 26 14:39:19 2010 @@ -11,7 +11,7 @@ from pypy.module.cpyext.api import ( generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, - METH_VARARGS) + METH_VARARGS, build_type_checkers) from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated @@ -89,6 +89,8 @@ def descr_method_repr(self): return self.getrepr(self.space, "built-in method '%s' of '%s' object" % (self.name, self.w_objclass.getname(self.space, '?'))) +PyMethod_Check, PyMethod_CheckExact = build_type_checkers("Method", W_PyCMethodObject) +PyCFunction_Check, PyCFunction_CheckExact = build_type_checkers("CFunction", W_PyCFunctionObject) class W_PyCWrapperObject(Wrappable): def __init__(self, space, pto, method_name, wrapper_func, wrapper_func_kwds, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 14:39:19 2010 @@ -3555,12 +3555,6 @@ """Same as PyMem_Free().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyMethod_Check(space, o): - """Return true if o is a method object (has type PyMethod_Type). The - parameter must not be NULL.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyMethod_New(space, func, self, class): """Return a new method object, with func being any callable object; this is the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Mon Apr 26 14:39:19 2010 @@ -34,6 +34,16 @@ } ''' ), + ('isCFunction', 'METH_O', + ''' + if(PyCFunction_Check(args)) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } + ''' + ), ]) assert mod.getarg_O(1) == 1 raises(TypeError, mod.getarg_O) @@ -46,3 +56,5 @@ assert mod.getarg_OLD(1) == 1 assert mod.getarg_OLD() is None assert mod.getarg_OLD(1, 2) == (1, 2) + + assert mod.isCFunction(mod.getarg_O) From afa at codespeak.net Mon Apr 26 14:47:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 14:47:15 +0200 (CEST) Subject: [pypy-svn] r74073 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426124715.DB454282B90@codespeak.net> Author: afa Date: Mon Apr 26 14:47:14 2010 New Revision: 74073 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Log: PyLong_AsDouble, PyLong_AsVoidPtr Modified: pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/longobject.py Mon Apr 26 14:47:14 2010 @@ -61,6 +61,13 @@ """Return a new PyLongObject object from v, or NULL on failure.""" return space.long(space.wrap(val)) + at cpython_api([PyObject], lltype.Float, error=-1.0) +def PyLong_AsDouble(space, w_long): + """Return a C double representation of the contents of pylong. If + pylong cannot be approximately represented as a double, an + OverflowError exception is raised and -1.0 will be returned.""" + return space.float_w(space.float(w_long)) + @cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject) def PyLong_FromString(space, str, pend, base): """Return a new PyLongObject based on the string value in str, which is @@ -87,3 +94,12 @@ If the integer is larger than LONG_MAX, a positive long integer is returned.""" return space.wrap(rffi.cast(ADDR, p)) + at cpython_api([PyObject], rffi.VOIDP_real, error=lltype.nullptr(rffi.VOIDP_real.TO)) +def PyLong_AsVoidPtr(space, w_long): + """Convert a Python integer or long integer pylong to a C void pointer. + If pylong cannot be converted, an OverflowError will be raised. This + is only assured to produce a usable void pointer for values created + with PyLong_FromVoidPtr(). + For values outside 0..LONG_MAX, both signed and unsigned integers are accepted.""" + return rffi.cast(rffi.VOIDP_real, space.uint_w(w_long)) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 14:47:14 2010 @@ -3362,25 +3362,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], {double}) -def PyLong_AsDouble(space, pylong): - """Return a C double representation of the contents of pylong. If - pylong cannot be approximately represented as a double, an - OverflowError exception is raised and -1.0 will be returned.""" - raise NotImplementedError - - at cpython_api([PyObject], {void*}) -def PyLong_AsVoidPtr(space, pylong): - """Convert a Python integer or long integer pylong to a C void pointer. - If pylong cannot be converted, an OverflowError will be raised. This - is only assured to produce a usable void pointer for values created - with PyLong_FromVoidPtr(). - - - - For values outside 0..LONG_MAX, both signed and unsigned integers are accepted.""" - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyMapping_Size(space, o): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_longobject.py Mon Apr 26 14:47:14 2010 @@ -1,6 +1,5 @@ - import sys - +from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.longobject import W_LongObject from pypy.module.cpyext.test.test_api import BaseApiTest @@ -35,7 +34,8 @@ def test_fromdouble(self, space, api): w_value = api.PyLong_FromDouble(-12.74) assert space.unwrap(w_value) == -12 - + assert api.PyLong_AsDouble(w_value) == -12 + def test_type_check(self, space, api): w_l = space.wrap(sys.maxint + 1) assert api.PyLong_Check(w_l) @@ -64,6 +64,11 @@ assert api.PyErr_Occurred() api.PyErr_Clear() + def test_as_voidptr(self, space, api): + w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO)) + assert space.unwrap(w_l) == 0L + assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP_real.TO) + class AppTestLongObject(AppTestCpythonExtensionBase): def test_fromlonglong(self): module = self.import_extension('foo', [ From afa at codespeak.net Mon Apr 26 14:51:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 14:51:20 +0200 (CEST) Subject: [pypy-svn] r74074 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426125120.1A389282B90@codespeak.net> Author: afa Date: Mon Apr 26 14:51:18 2010 New Revision: 74074 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Ensure that functions are defined only once Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 14:51:18 2010 @@ -159,6 +159,9 @@ api_function = ApiFunction(argtypes, restype, func, borrowed, error) func.api_func = api_function + assert func_name not in FUNCTIONS + assert func_name not in FUNCTIONS_STATIC + if error is _NOT_SPECIFIED: raise ValueError("function %s has no return value for exceptions" % func) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Mon Apr 26 14:51:18 2010 @@ -66,16 +66,6 @@ return unicodedb.islinebreak(ord(ch)) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISSPACE(space, ch): - """Return 1 or 0 depending on whether ch is a whitespace character.""" - return unicodedb.isspace(ord(ch)) - - at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) -def Py_UNICODE_ISALNUM(space, ch): - """Return 1 or 0 depending on whether ch is an alphanumeric character.""" - return unicodedb.isalnum(ord(ch)) - - at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISDECIMAL(space, ch): """Return 1 or 0 depending on whether ch is a decimal character.""" return unicodedb.isdecimal(ord(ch)) From afa at codespeak.net Mon Apr 26 15:08:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 15:08:38 +0200 (CEST) Subject: [pypy-svn] r74075 - pypy/trunk/lib-python/modified-2.5.2 Message-ID: <20100426130838.2DAB0282B90@codespeak.net> Author: afa Date: Mon Apr 26 15:08:36 2010 New Revision: 74075 Added: pypy/trunk/lib-python/modified-2.5.2/locale.py - copied, changed from r74074, pypy/trunk/lib-python/2.5.2/locale.py Log: Port CPython's issue6202: remove specific mac cases, darwin is a posix platform. Copied: pypy/trunk/lib-python/modified-2.5.2/locale.py (from r74074, pypy/trunk/lib-python/2.5.2/locale.py) ============================================================================== --- pypy/trunk/lib-python/2.5.2/locale.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/locale.py Mon Apr 26 15:08:36 2010 @@ -487,10 +487,8 @@ """ _setlocale(category, _build_localename(getdefaultlocale())) -if sys.platform in ('win32', 'darwin', 'mac'): +if sys.platform == 'win32': # On Win32, this will return the ANSI code page - # On the Mac, it should return the system encoding; - # it might return "ascii" instead def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using.""" import _locale From afa at codespeak.net Mon Apr 26 16:59:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 16:59:30 +0200 (CEST) Subject: [pypy-svn] r74079 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426145930.ACC23282B9C@codespeak.net> Author: afa Date: Mon Apr 26 16:59:28 2010 New Revision: 74079 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Log: PyNumber_Int Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Mon Apr 26 16:59:28 2010 @@ -38,6 +38,13 @@ return space.int_w(w_obj) #XXX: this is wrong on win64 @cpython_api([PyObject], PyObject) +def PyNumber_Int(space, w_obj): + """Returns the o converted to an integer object on success, or NULL on failure. + If the argument is outside the integer range a long object will be returned + instead. This is the equivalent of the Python expression int(o).""" + return space.int(w_obj) + + at cpython_api([PyObject], PyObject) def PyNumber_Long(space, w_obj): """ Returns the o converted to a long integer object on success, or NULL on failure. This is the equivalent of the Python expression long(o).""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 16:59:28 2010 @@ -3693,17 +3693,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyNumber_Int(space, o): - """ - - - - Returns the o converted to an integer object on success, or NULL on failure. - If the argument is outside the integer range a long object will be returned - instead. This is the equivalent of the Python expression int(o).""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyNumber_Index(space, o): """Returns the o converted to a Python int or long on success or NULL with a TypeError exception raised on failure. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_number.py Mon Apr 26 16:59:28 2010 @@ -20,6 +20,10 @@ w_l = api.PyNumber_Long(space.wrap(123)) assert api.PyLong_CheckExact(w_l) + def test_number_int(self, space, api): + w_l = api.PyNumber_Int(space.wrap(123L)) + assert api.PyInt_CheckExact(w_l) + def test_numbermethods(self, space, api): assert "ab" == space.unwrap( api.PyNumber_Add(space.wrap("a"), space.wrap("b"))) From agaynor at codespeak.net Mon Apr 26 17:26:54 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 26 Apr 2010 17:26:54 +0200 (CEST) Subject: [pypy-svn] r74080 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100426152654.CB45E282B9C@codespeak.net> Author: agaynor Date: Mon Apr 26 17:26:53 2010 New Revision: 74080 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: Implemented CPython's string search algorithm and switch lltype rstr to use it. This is about 55% on stringbench for strs, and faster for unicode as well. Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Mon Apr 26 17:26:53 2010 @@ -55,7 +55,7 @@ return (llmemory.offsetof(TP, 'chars') + llmemory.itemoffsetof(TP.chars, 0) + llmemory.sizeof(CHAR_TP) * item) - + def copy_string_contents(src, dst, srcstart, dststart, length): assert srcstart >= 0 assert dststart >= 0 @@ -121,7 +121,7 @@ AbstractStringRepr.__init__(self, *args) self.ll = LLHelpers self.malloc = mallocstr - + def ll_decode_latin1(self, value): lgt = len(value.chars) s = mallocunicode(lgt) @@ -235,6 +235,25 @@ j = 0 return T + +FAST_COUNT = 0 +FAST_FIND = 1 +FAST_RFIND = 2 + + +# XXX: This should be set to the number of bits in a long. Having a lower +# value here doesn't break anything, it just decreases the accuracy of the +# bloom filter heuristic, which results in a worse runtime (but correct results) +BLOOM_WIDTH = 32 + + +def bloom_add(mask, c): + return mask | (1 << (ord(c) & (BLOOM_WIDTH - 1))) + +def bloom(mask, c): + return mask & (1 << (ord(c) & (BLOOM_WIDTH - 1))) + + class LLHelpers(AbstractLLHelpers): @purefunction @@ -493,126 +512,132 @@ i += 1 return count + @classmethod @purefunction def ll_find(cls, s1, s2, start, end): - """Knuth Morris Prath algorithm for substring match""" - len2 = len(s2.chars) - if len2 == 1: - return cls.ll_find_char(s1, s2.chars[0], start, end) - len1 = len(s1.chars) - if end > len1: - end = len1 - if len2 == 0: - if (end-start) < 0: - return -1 - return start - - T = ll_construct_restart_positions(s2, len2) + if start < 0: + start = 0 + if end > len(s1.chars): + end = len(s1.chars) + if end - start < 0: + return -1 - # Now the find algorithm - i = 0 - m = start - while m+i len(s1.chars): end = len(s1.chars) - if len2 == 0: - if (end-start) < 0: - return -1 - return end - # Construct the array of possible restarting positions - T = malloc( SIGNED_ARRAY, len2 ) - T[0] = 1 - i = 1 - j = 1 - while i1: - j = T[j-2] - else: - T[i] = 1 - i += 1 - j = 1 + if end - start < 0: + return -1 - # Now the find algorithm - i = 1 - m = end - while m-i>=start: - if s1.chars[m-i]==s2.chars[len2-i]: - if i==len2: - return m-i - i += 1 - else: - # mismatch, go back to the last possible starting pos - if i==1: - m -= 1 - else: - e = T[i-2] - m = m - i + e - i = e - return -1 - ll_rfind = classmethod(ll_rfind) + m = len(s2.chars) + if m == 0: + return end + elif m == 1: + return cls.ll_rfind_char(s1, s2.chars[0], start, end) + + return cls.ll_search(s1, s2, start, end, FAST_RFIND) + @classmethod @purefunction def ll_count(cls, s1, s2, start, end): - """Knuth Morris Prath algorithm for substring match""" - # XXX more code should be shared with ll_find - len1 = len(s1.chars) - if end > len1: - end = len1 - len2 = len(s2.chars) - if len2 == 1: - return cls.ll_count_char(s1, s2.chars[0], start, end) - if len2 == 0: - if (end-start) < 0: - return 0 + if start < 0: + start = 0 + if end > len(s1.chars): + end = len(s1.chars) + if end - start < 0: + return 0 + + m = len(s2.chars) + if m == 0: return end - start + 1 - T = ll_construct_restart_positions(s2, len2) + elif m == 1: + return cls.ll_count_char(s1, s2.chars[0], start, end) + + return cls.ll_search(s1, s2, start, end, FAST_COUNT) - # Now the find algorithm - i = 0 - m = start - result = 0 - while m+i= start: + i -= 1 + if s1.chars[i] == s2.chars[0]: + for j in xrange(mlast, 0, -1): + if s1.chars[i+j] != s2.chars[j]: + break + else: + return i + if i-1 >= 0 and not bloom(mask, s1.chars[i-1]): + i -= m + else: + i -= skip + else: + if i-1 >= 0 and not bloom(mask, s1.chars[i-1]): + i -= m + + if mode != FAST_COUNT: + return -1 + return count def ll_join_strs(length, items): num_items = length @@ -904,7 +929,7 @@ self.ll_strnext = ll_strnext class StringIteratorRepr(BaseStringIteratorRepr): - + lowleveltype = Ptr(GcStruct('stringiter', ('string', string_repr.lowleveltype), ('index', Signed))) @@ -949,4 +974,3 @@ unboxed_instance_str_prefix = string_repr.convert_const("") - From arigo at codespeak.net Mon Apr 26 18:44:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Apr 2010 18:44:02 +0200 (CEST) Subject: [pypy-svn] r74084 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100426164402.B0010282B9C@codespeak.net> Author: arigo Date: Mon Apr 26 18:44:01 2010 New Revision: 74084 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: getfield. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Mon Apr 26 18:44:01 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.jit.metainterp.history import getkind -from pypy.objspace.flow.model import SpaceOperation +from pypy.objspace.flow.model import SpaceOperation, Variable, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind @@ -26,20 +26,44 @@ for block in graph.iterblocks(): rename = {} newoperations = [] + if block.exitswitch == c_last_exception: + op_raising_exception = block.operations[-1] + else: + op_raising_exception = Ellipsis for op in block.operations: - for i, v in enumerate(op.args): - if v in rename: - op = SpaceOperation(op.opname, op.args[:], - op.result) - op.args[i] = rename[v] try: - newoperations.append(self.rewrite_operation(op)) + op1 = self.rewrite_operation(op) except NoOp: if op.result is not None: rename[op.result] = rename.get(op.args[0], op.args[0]) + if op is op_raising_exception: + self.killed_exception_raising_operation(block) + else: + op2 = self.do_renaming(rename, op1) + newoperations.append(op2) block.operations = newoperations self.optimize_goto_if_not(block) + def do_renaming(self, rename, op): + op = SpaceOperation(op.opname, op.args[:], op.result) + for i, v in enumerate(op.args): + if isinstance(v, Variable): + if v in rename: + op.args[i] = rename[v] + elif isinstance(v, ListOfKind): + newlst = [] + for x in v: + if x in rename: + x = rename[x] + newlst.append(x) + op.args[i] = ListOfKind(v.kind, newlst) + return op + + def killed_exception_raising_operation(self, block): + assert block.exits[0].exitcase is None + del block.exits[1:] + block.exitswitch = None + # ---------- def optimize_goto_if_not(self, block): @@ -81,6 +105,7 @@ def rewrite_op_cast_int_to_unichar(self, op): raise NoOp def rewrite_op_cast_char_to_int(self, op): raise NoOp def rewrite_op_cast_unichar_to_int(self, op): raise NoOp + def rewrite_op_cast_pointer(self, op): raise NoOp def rewrite_op_direct_call(self, op): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' @@ -164,9 +189,64 @@ #else: return ARRAY.OF == lltype.Void + def rewrite_op_getfield(self, op): + #if self.is_typeptr_getset(op): + # self.handle_getfield_typeptr(op) + # return + # turn the flow graph 'getfield' operation into our own version + [v_inst, c_fieldname] = op.args + RESULT = op.result.concretetype + if RESULT is lltype.Void: + raise NoOp + # check for virtualizable + #try: + # if self.is_virtualizable_getset(op): + # vinfo = self.codewriter.metainterp_sd.virtualizable_info + # index = vinfo.static_field_to_extra_box[op.args[1].value] + # self.emit('getfield_vable', + # self.var_position(v_inst), + # index) + # self.register_var(op.result) + # return + #except VirtualizableArrayField: + # # xxx hack hack hack + # vinfo = self.codewriter.metainterp_sd.virtualizable_info + # arrayindex = vinfo.array_field_counter[op.args[1].value] + # self.vable_array_vars[op.result] = (op.args[0], arrayindex) + # return + # check for deepfrozen structures that force constant-folding + hints = v_inst.concretetype.TO._hints + accessor = hints.get("immutable_fields") + if accessor and c_fieldname.value in accessor.fields: + pure = '_pure' + if accessor.fields[c_fieldname.value] == "[*]": + self.immutable_arrays[op.result] = True + elif hints.get('immutable'): + pure = '_pure' + else: + pure = '' + argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') + descr = self.cpu.fielddescrof(v_inst.concretetype.TO, + c_fieldname.value) + if isinstance(RESULT, lltype.Primitive): + kind = primitive_type_size[RESULT] + else: + kind = getkind(RESULT)[0] + return SpaceOperation('getfield_%s_%s%s' % (argname, kind, pure), + [v_inst, descr], op.result) + # ____________________________________________________________ _rewrite_ops = {} for _name in dir(Transformer): if _name.startswith('rewrite_op_'): _rewrite_ops[_name[len('rewrite_op_'):]] = getattr(Transformer, _name) + +primitive_type_size = { + lltype.Signed: 'i', + lltype.Unsigned: 'i', + lltype.Bool: 'c', + lltype.Char: 'c', + lltype.UniChar: 'u', + lltype.Float: 'f', + } Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Mon Apr 26 18:44:01 2010 @@ -9,6 +9,8 @@ class FakeCPU: def calldescrof(self, FUNC, ARGS, RESULT): return ('calldescr', FUNC, ARGS, RESULT) + def fielddescrof(self, STRUCT, name): + return ('fielddescr', STRUCT, name) class FakeLink: args = [] @@ -100,3 +102,33 @@ for v in op.args[1:]: kind = getkind(v.concretetype) assert kind == 'void' or kind[0] in expectedkind + +def test_getfield(): + # XXX a more compact encoding would be possible, something along + # the lines of getfield_gc_r %r0, $offset, %r1 + # which would not need a Descr at all. + S1 = lltype.Struct('S1') + S2 = lltype.GcStruct('S2') + S = lltype.GcStruct('S', ('int', lltype.Signed), + ('ps1', lltype.Ptr(S1)), + ('ps2', lltype.Ptr(S2)), + ('flt', lltype.Float), + ('boo', lltype.Bool), + ('chr', lltype.Char), + ('unc', lltype.UniChar)) + for name, suffix in [('int', 'i'), + ('ps1', 'i'), + ('ps2', 'r'), + ('flt', 'f'), + ('boo', 'c'), + ('chr', 'c'), + ('unc', 'u')]: + v_parent = varoftype(lltype.Ptr(S)) + c_name = Constant(name, lltype.Void) + v_result = varoftype(getattr(S, name)) + op = SpaceOperation('getfield', [v_parent, c_name], v_result) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'getfield_gc_' + suffix + fielddescr = ('fielddescr', S, name) + assert op1.args == [v_parent, fielddescr] + assert op1.result == v_result From arigo at codespeak.net Mon Apr 26 18:44:23 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Apr 2010 18:44:23 +0200 (CEST) Subject: [pypy-svn] r74085 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph metainterp Message-ID: <20100426164423.08152282B9D@codespeak.net> Author: arigo Date: Mon Apr 26 18:44:22 2010 New Revision: 74085 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Log: More getfield. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Mon Apr 26 18:44:22 2010 @@ -1229,8 +1229,8 @@ ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) return getattr(ptr, fieldname) -def do_getfield_gc_int(struct, fieldnum, memocast): - return cast_to_int(_getfield_gc(struct, fieldnum), memocast) +def do_getfield_gc_int(struct, fieldnum): + return cast_to_int(_getfield_gc(struct, fieldnum)) def do_getfield_gc_float(struct, fieldnum): return cast_to_float(_getfield_gc(struct, fieldnum)) @@ -1238,19 +1238,19 @@ def do_getfield_gc_ptr(struct, fieldnum): return cast_to_ptr(_getfield_gc(struct, fieldnum)) -def _getfield_raw(struct, fieldnum, memocast): +def _getfield_raw(struct, fieldnum): STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct, memocast) + ptr = cast_from_int(lltype.Ptr(STRUCT), struct) return getattr(ptr, fieldname) -def do_getfield_raw_int(struct, fieldnum, memocast): - return cast_to_int(_getfield_raw(struct, fieldnum, memocast), memocast) +def do_getfield_raw_int(struct, fieldnum): + return cast_to_int(_getfield_raw(struct, fieldnum)) -def do_getfield_raw_float(struct, fieldnum, memocast): - return cast_to_float(_getfield_raw(struct, fieldnum, memocast)) +def do_getfield_raw_float(struct, fieldnum): + return cast_to_float(_getfield_raw(struct, fieldnum)) -def do_getfield_raw_ptr(struct, fieldnum, memocast): - return cast_to_ptr(_getfield_raw(struct, fieldnum, memocast)) +def do_getfield_raw_ptr(struct, fieldnum): + return cast_to_ptr(_getfield_raw(struct, fieldnum)) def do_new(size): TYPE = symbolic.Size2Type[size] Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Mon Apr 26 18:44:22 2010 @@ -371,39 +371,63 @@ else: raise NotImplementedError - def do_getfield_gc(self, structbox, fielddescr): +## def do_getfield_gc(self, structbox, fielddescr): +## assert isinstance(fielddescr, Descr) +## struct = structbox.getref_base() +## if fielddescr.typeinfo == REF: +## return history.BoxPtr(llimpl.do_getfield_gc_ptr(struct, +## fielddescr.ofs)) +## elif fielddescr.typeinfo == INT: +## return history.BoxInt(llimpl.do_getfield_gc_int(struct, +## fielddescr.ofs, +## self.memo_cast)) +## elif fielddescr.typeinfo == FLOAT: +## return history.BoxFloat(llimpl.do_getfield_gc_float(struct, +## fielddescr.ofs)) +## else: +## raise NotImplementedError + + def bh_getfield_gc_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) - struct = structbox.getref_base() - if fielddescr.typeinfo == REF: - return history.BoxPtr(llimpl.do_getfield_gc_ptr(struct, - fielddescr.ofs)) - elif fielddescr.typeinfo == INT: - return history.BoxInt(llimpl.do_getfield_gc_int(struct, - fielddescr.ofs, - self.memo_cast)) - elif fielddescr.typeinfo == FLOAT: - return history.BoxFloat(llimpl.do_getfield_gc_float(struct, - fielddescr.ofs)) - else: - raise NotImplementedError + return llimpl.do_getfield_gc_int(struct, fielddescr.ofs) + bh_getfield_gc_c = bh_getfield_gc_i + bh_getfield_gc_u = bh_getfield_gc_i + def bh_getfield_gc_r(self, struct, fielddescr): + assert isinstance(fielddescr, Descr) + return llimpl.do_getfield_gc_ptr(struct, fielddescr.ofs) + def bh_getfield_gc_f(self, struct, fielddescr): + assert isinstance(fielddescr, Descr) + return llimpl.do_getfield_gc_float(struct, fielddescr.ofs) - def do_getfield_raw(self, structbox, fielddescr): +## def do_getfield_raw(self, structbox, fielddescr): +## assert isinstance(fielddescr, Descr) +## struct = self.cast_int_to_adr(structbox.getint()) +## if fielddescr.typeinfo == REF: +## return history.BoxPtr(llimpl.do_getfield_raw_ptr(struct, +## fielddescr.ofs, +## self.memo_cast)) +## elif fielddescr.typeinfo == INT: +## return history.BoxInt(llimpl.do_getfield_raw_int(struct, +## fielddescr.ofs, +## self.memo_cast)) +## elif fielddescr.typeinfo == FLOAT: +## return history.BoxFloat(llimpl.do_getfield_raw_float(struct, +## fielddescr.ofs, +## self.memo_cast)) +## else: +## raise NotImplementedError + + def bh_getfield_raw_i(self, struct, fielddescr): assert isinstance(fielddescr, Descr) - struct = self.cast_int_to_adr(structbox.getint()) - if fielddescr.typeinfo == REF: - return history.BoxPtr(llimpl.do_getfield_raw_ptr(struct, - fielddescr.ofs, - self.memo_cast)) - elif fielddescr.typeinfo == INT: - return history.BoxInt(llimpl.do_getfield_raw_int(struct, - fielddescr.ofs, - self.memo_cast)) - elif fielddescr.typeinfo == FLOAT: - return history.BoxFloat(llimpl.do_getfield_raw_float(struct, - fielddescr.ofs, - self.memo_cast)) - else: - raise NotImplementedError + return llimpl.do_getfield_raw_int(struct, fielddescr.ofs) + bh_getfield_raw_c = bh_getfield_raw_i + bh_getfield_raw_u = bh_getfield_raw_i + def bh_getfield_raw_r(self, struct, fielddescr): + assert isinstance(fielddescr, Descr) + return llimpl.do_getfield_raw_ptr(struct, fielddescr.ofs) + def bh_getfield_raw_f(self, struct, fielddescr): + assert isinstance(fielddescr, Descr) + return llimpl.do_getfield_raw_float(struct, fielddescr.ofs) def do_new(self, size): assert isinstance(size, Descr) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Mon Apr 26 18:44:22 2010 @@ -183,10 +183,32 @@ def do_getfield_gc(self, structbox, fielddescr): raise NotImplementedError - + + def bh_getfield_gc_i(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_gc_c(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_gc_u(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_gc_r(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_gc_f(self, struct, fielddescr): + raise NotImplementedError + def do_getfield_raw(self, structbox, fielddescr): raise NotImplementedError + def bh_getfield_raw_i(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_raw_c(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_raw_u(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_raw_r(self, struct, fielddescr): + raise NotImplementedError + def bh_getfield_raw_f(self, struct, fielddescr): + raise NotImplementedError + def do_new(self, sizedescr): raise NotImplementedError @@ -198,7 +220,7 @@ def bh_new_array(self, arraydescr, length): raise NotImplementedError - + def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Mon Apr 26 18:44:22 2010 @@ -224,6 +224,10 @@ return a - b @arguments("i", "i", returns="i") + def opimpl_int_mul(self, a, b): + return a * b + + @arguments("i", "i", returns="i") def opimpl_uint_floordiv(self, a, b): c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) return intmask(c) @@ -351,3 +355,47 @@ @arguments("d", "r", "i", "r") def opimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) + + @arguments("r", "d", returns="i") + def opimpl_getfield_gc_i(self, struct, fielddescr): + return self.cpu.bh_getfield_gc_i(struct, fielddescr) + @arguments("r", "d", returns="i") + def opimpl_getfield_gc_c(self, struct, fielddescr): + return self.cpu.bh_getfield_gc_c(struct, fielddescr) + @arguments("r", "d", returns="i") + def opimpl_getfield_gc_u(self, struct, fielddescr): + return self.cpu.bh_getfield_gc_u(struct, fielddescr) + @arguments("r", "d", returns="r") + def opimpl_getfield_gc_r(self, struct, fielddescr): + return self.cpu.bh_getfield_gc_r(struct, fielddescr) + @arguments("r", "d", returns="f") + def opimpl_getfield_gc_f(self, struct, fielddescr): + return self.cpu.bh_getfield_gc_f(struct, fielddescr) + + opimpl_getfield_gc_i_pure = opimpl_getfield_gc_i + opimpl_getfield_gc_c_pure = opimpl_getfield_gc_c + opimpl_getfield_gc_u_pure = opimpl_getfield_gc_u + opimpl_getfield_gc_r_pure = opimpl_getfield_gc_r + opimpl_getfield_gc_f_pure = opimpl_getfield_gc_f + + @arguments("r", "d", returns="i") + def opimpl_getfield_raw_i(self, struct, fielddescr): + return self.cpu.bh_getfield_raw_i(struct, fielddescr) + @arguments("r", "d", returns="i") + def opimpl_getfield_raw_c(self, struct, fielddescr): + return self.cpu.bh_getfield_raw_c(struct, fielddescr) + @arguments("r", "d", returns="i") + def opimpl_getfield_raw_u(self, struct, fielddescr): + return self.cpu.bh_getfield_raw_u(struct, fielddescr) + @arguments("r", "d", returns="r") + def opimpl_getfield_raw_r(self, struct, fielddescr): + return self.cpu.bh_getfield_raw_r(struct, fielddescr) + @arguments("r", "d", returns="f") + def opimpl_getfield_raw_f(self, struct, fielddescr): + return self.cpu.bh_getfield_raw_f(struct, fielddescr) + + opimpl_getfield_raw_i_pure = opimpl_getfield_raw_i + opimpl_getfield_raw_c_pure = opimpl_getfield_raw_c + opimpl_getfield_raw_u_pure = opimpl_getfield_raw_u + opimpl_getfield_raw_r_pure = opimpl_getfield_raw_r + opimpl_getfield_raw_f_pure = opimpl_getfield_raw_f Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/warmspot.py Mon Apr 26 18:44:22 2010 @@ -145,6 +145,7 @@ def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, optimizer=None, **kwds): + import py; py.test.skip("XXX") pyjitpl._warmrunnerdesc = self # this is a global for debugging only! if policy is None: policy = JitPolicy() From afa at codespeak.net Mon Apr 26 18:56:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 18:56:20 +0200 (CEST) Subject: [pypy-svn] r74086 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426165620.749BD282B9C@codespeak.net> Author: afa Date: Mon Apr 26 18:56:18 2010 New Revision: 74086 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Let stubs.py import, even if it produces invalid code. Remove functions already implemented. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 18:56:18 2010 @@ -1,56 +1,37 @@ - - at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) -def _PyObject_NewVar(space, type, size): - """This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError +from pypy.module.cpyext.api import ( + cpython_api, PyObject, PyObjectP, CANNOT_FAIL + ) +from pypy.rpython.lltypesystem import rffi, lltype + +# we don't really care +PyTypeObjectPtr = lltype.Void +Py_ssize_t = lltype.Void +PyMethodDef = lltype.Void +PyGetSetDef = lltype.Void +PyMemberDef = lltype.Void +Py_buffer = lltype.Void +Py_complex = lltype.Void +va_list = lltype.Void +PyDateTime_Date = lltype.Void +PyDateTime_DateTime = lltype.Void +PyDateTime_Time = lltype.Void +wrapperbase = lltype.Void +FILE = lltype.Void +PyFileObject = lltype.Void +PyCodeObject = lltype.Void +PyFrameObject = lltype.Void +_inittab = lltype.Void +PyThreadState = lltype.Void +PyInterpreterState = lltype.Void +PyOS_sighandler_t = lltype.Void +Py_UNICODE = lltype.Void +PyCompilerFlags = lltype.Void +_node = lltype.Void @cpython_api([PyObject], lltype.Void) def _PyObject_Del(space, op): raise NotImplementedError - at cpython_api([PyObject, PyTypeObjectPtr], PyObject, borrowed=True) -def PyObject_Init(space, op, type): - """Initialize a newly-allocated object op with its type and initial - reference. Returns the initialized object. If type indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected.""" - raise NotImplementedError - - at cpython_api([PyObject, PyTypeObjectPtr, Py_ssize_t], PyObject, borrowed=True) -def PyObject_InitVar(space, op, type, size): - """This does everything PyObject_Init() does, and also initializes the - length information for a variable-size object. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([{TYPE}, PyTypeObjectPtr], {TYPE*}) -def PyObject_New(space, , type): - """Allocate a new Python object using the C structure type TYPE and the - Python type object type. Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the tp_basicsize field of - the type object.""" - raise NotImplementedError - - at cpython_api([{TYPE}, PyTypeObjectPtr, Py_ssize_t], {TYPE*}) -def PyObject_NewVar(space, , type, size): - """Allocate a new Python object using the C structure type TYPE and the - Python type object type. Fields not defined by the Python object header - are not initialized. The allocated memory allows for the TYPE structure - plus size fields of the size given by the tp_itemsize field of - type. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, PyMethodDef], PyObject, borrowed=True) def Py_InitModule(space, name, methods): """Create a new module object based on a name and table of functions, @@ -70,33 +51,33 @@ methods argument.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ...], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, ], rffi.INT_real, error=0) def PyArg_ParseTuple(space, args, format, ): """Parse the parameters of a function that takes only positional parameters into local variables. Returns true on success; on failure, it returns false and raises the appropriate exception.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, {va_list}], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, va_list], rffi.INT_real, error=0) def PyArg_VaParse(space, args, format, vargs): """Identical to PyArg_ParseTuple(), except that it accepts a va_list rather than a variable number of arguments.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, ...], rffi.INT_real) -def PyArg_ParseTupleAndKeywords(space, args, kw, format, keywords[], ): + at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, ], rffi.INT_real, error=0) +def PyArg_ParseTupleAndKeywords(space, args, kw, format, keywords, ): """Parse the parameters of a function that takes both positional and keyword parameters into local variables. Returns true on success; on failure, it returns false and raises the appropriate exception.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, {va_list}], rffi.INT_real) -def PyArg_VaParseTupleAndKeywords(space, args, kw, format, keywords[], vargs): + at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, va_list], rffi.INT_real, error=0) +def PyArg_VaParseTupleAndKeywords(space, args, kw, format, keywords, vargs): """Identical to PyArg_ParseTupleAndKeywords(), except that it accepts a va_list rather than a variable number of arguments.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ...], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, ], rffi.INT_real, error=0) def PyArg_Parse(space, args, format, ): """Function used to deconstruct the argument lists of "old-style" functions --- these are functions which use the METH_OLDARGS parameter @@ -107,7 +88,7 @@ purpose.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, Py_ssize_t, Py_ssize_t, ...], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, Py_ssize_t, Py_ssize_t, ], rffi.INT_real, error=0) def PyArg_UnpackTuple(space, args, name, min, max, ): """A simpler form of parameter retrieval which does not use a format string to specify the types of the arguments. Functions which use this method to @@ -150,7 +131,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, ...], PyObject) + at cpython_api([rffi.CCHARP, ], PyObject) def Py_BuildValue(space, format, ): """Create a new value based on a format string similar to those accepted by the PyArg_Parse*() family of functions and a sequence of values. @@ -324,18 +305,18 @@ is set and NULL returned.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {va_list}], PyObject) + at cpython_api([rffi.CCHARP, va_list], PyObject) def Py_VaBuildValue(space, format, vargs): """Identical to Py_BuildValue(), except that it accepts a va_list rather than a variable number of arguments.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=0) def PyObject_CheckBuffer(space, obj): """Return 1 if obj supports the buffer interface otherwise 0.""" raise NotImplementedError - at cpython_api([PyObject, {Py_buffer*}, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, Py_buffer, rffi.INT_real], rffi.INT_real, error=-1) def PyObject_GetBuffer(space, obj, view, flags): """Export obj into a Py_buffer, view. These arguments must never be NULL. The flags argument is a bit field indicating what @@ -473,19 +454,19 @@ This is equivalent to (PyBUF_ND).""" raise NotImplementedError - at cpython_api([{Py_buffer*}], lltype.Void) + at cpython_api([Py_buffer], lltype.Void) def PyBuffer_Release(space, view): """Release the buffer view. This should be called when the buffer is no longer being used as it may free memory from it.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], Py_ssize_t) -def PyBuffer_SizeFromFormat(space, ): + at cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL) +def PyBuffer_SizeFromFormat(space, format): """Return the implied ~Py_buffer.itemsize from the struct-stype ~Py_buffer.format.""" raise NotImplementedError - at cpython_api([PyObject, {void*}, Py_ssize_t, lltype.Char], rffi.INT_real) + at cpython_api([PyObject, rffi.VOIDP_real, Py_ssize_t, lltype.Char], rffi.INT_real, error=-1) def PyObject_CopyToObject(space, obj, buf, len, fortran): """Copy len bytes of data pointed to by the contiguous chunk of memory pointed to by buf into the buffer exported by obj. The buffer must of @@ -499,7 +480,7 @@ matter and the copy will be made in whatever way is more efficient.""" raise NotImplementedError - at cpython_api([{Py_buffer*}, lltype.Char], rffi.INT_real) + at cpython_api([Py_buffer, lltype.Char], rffi.INT_real, error=-1) def PyBuffer_IsContiguous(space, view, fortran): """Return 1 if the memory defined by the view is C-style (fortran is 'C') or Fortran-style (fortran is 'F') contiguous or either one @@ -513,7 +494,7 @@ given shape with the given number of bytes per element.""" raise NotImplementedError - at cpython_api([{Py_buffer*}, PyObject, {void*}, Py_ssize_t, rffi.INT_real, rffi.INT_real], rffi.INT_real) + at cpython_api([Py_buffer, PyObject, rffi.VOIDP_real, Py_ssize_t, rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1) def PyBuffer_FillInfo(space, view, obj, buf, len, readonly, infoflags): """Fill in a buffer-info structure, view, correctly for an exporter that can only share a contiguous chunk of memory of "unsigned bytes" of the given @@ -525,7 +506,7 @@ """Return a memoryview object from an object that defines the buffer interface.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyBuffer_Check(space, p): """Return true if the argument has type PyBuffer_Type.""" raise NotImplementedError @@ -557,7 +538,7 @@ systems.""" raise NotImplementedError - at cpython_api([{void*}, Py_ssize_t], PyObject) + at cpython_api([rffi.VOIDP_real, Py_ssize_t], PyObject) def PyBuffer_FromMemory(space, ptr, size): """Return a new read-only buffer object that reads from a specified location in memory, with a specified size. The caller is responsible for ensuring @@ -570,7 +551,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{void*}, Py_ssize_t], PyObject) + at cpython_api([rffi.VOIDP_real, Py_ssize_t], PyObject) def PyBuffer_FromReadWriteMemory(space, ptr, size): """Similar to PyBuffer_FromMemory(), but the returned buffer is writable. @@ -590,13 +571,13 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyByteArray_Check(space, o): """Return true if the object o is a bytearray object or an instance of a subtype of the bytearray type.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyByteArray_CheckExact(space, o): """Return true if the object o is a bytearray object, but not an instance of a subtype of the bytearray type.""" @@ -626,155 +607,29 @@ """Return the size of bytearray after checking for a NULL pointer.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=lltype.nullptr(rffi.CCHARP.TO)) def PyByteArray_AsString(space, bytearray): """Return the contents of bytearray as a char array after checking for a NULL pointer.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) def PyByteArray_Resize(space, bytearray, len): """Resize the internal buffer of bytearray to len.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyByteArray_AS_STRING(space, bytearray): """Macro version of PyByteArray_AsString().""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) + at cpython_api([PyObject], Py_ssize_t, error=-1) def PyByteArray_GET_SIZE(space, bytearray): """Macro version of PyByteArray_Size().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyCapsule_CheckExact(space, p): - """Return true if its argument is a PyCapsule.""" - raise NotImplementedError - - at cpython_api([{void*}, rffi.CCHARP, {PyCapsule_Destructor}], PyObject) -def PyCapsule_New(space, pointer, name, destructor): - """Create a PyCapsule encapsulating the pointer. The pointer - argument may not be NULL. - - On failure, set an exception and return NULL. - - The name string may either be NULL or a pointer to a valid C string. If - non-NULL, this string must outlive the capsule. (Though it is permitted to - free it inside the destructor.) - - If the destructor argument is not NULL, it will be called with the - capsule as its argument when it is destroyed. - - If this capsule will be stored as an attribute of a module, the name should - be specified as modulename.attributename. This will enable other modules - to import the capsule using PyCapsule_Import().""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], {void*}) -def PyCapsule_GetPointer(space, capsule, name): - """Retrieve the pointer stored in the capsule. On failure, set an exception - and return NULL. - - The name parameter must compare exactly to the name stored in the capsule. - If the name stored in the capsule is NULL, the name passed in must also - be NULL. Python uses the C function strcmp() to compare capsule - names.""" - raise NotImplementedError - - at cpython_api([PyObject], {PyCapsule_Destructor}) -def PyCapsule_GetDestructor(space, capsule): - """Return the current destructor stored in the capsule. On failure, set an - exception and return NULL. - - It is legal for a capsule to have a NULL destructor. This makes a NULL - return code somewhat ambiguous; use PyCapsule_IsValid() or - PyErr_Occurred() to disambiguate.""" - raise NotImplementedError - - at cpython_api([PyObject], {void*}) -def PyCapsule_GetContext(space, capsule): - """Return the current context stored in the capsule. On failure, set an - exception and return NULL. - - It is legal for a capsule to have a NULL context. This makes a NULL - return code somewhat ambiguous; use PyCapsule_IsValid() or - PyErr_Occurred() to disambiguate.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP) -def PyCapsule_GetName(space, capsule): - """Return the current name stored in the capsule. On failure, set an exception - and return NULL. - - It is legal for a capsule to have a NULL name. This makes a NULL return - code somewhat ambiguous; use PyCapsule_IsValid() or - PyErr_Occurred() to disambiguate.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, rffi.INT_real], {void*}) -def PyCapsule_Import(space, name, no_block): - """Import a pointer to a C object from a capsule attribute in a module. The - name parameter should specify the full name to the attribute, as in - module.attribute. The name stored in the capsule must match this - string exactly. If no_block is true, import the module without blocking - (using PyImport_ImportModuleNoBlock()). If no_block is false, - import the module conventionally (using PyImport_ImportModule()). - - Return the capsule's internal pointer on success. On failure, set an - exception and return NULL. However, if PyCapsule_Import() failed to - import the module, and no_block was true, no exception is set.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) -def PyCapsule_IsValid(space, capsule, name): - """Determines whether or not capsule is a valid capsule. A valid capsule is - non-NULL, passes PyCapsule_CheckExact(), has a non-NULL pointer - stored in it, and its internal name matches the name parameter. (See - PyCapsule_GetPointer() for information on how capsule names are - compared.) - - In other words, if PyCapsule_IsValid() returns a true value, calls to - any of the accessors (any function starting with PyCapsule_Get()) are - guaranteed to succeed. - - Return a nonzero value if the object is valid and matches the name passed in. - Return 0 otherwise. This function will not fail.""" - raise NotImplementedError - - at cpython_api([PyObject, {void*}], rffi.INT_real) -def PyCapsule_SetContext(space, capsule, context): - """Set the context pointer inside capsule to context. - - Return 0 on success. Return nonzero and set an exception on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, {PyCapsule_Destructor}], rffi.INT_real) -def PyCapsule_SetDestructor(space, capsule, destructor): - """Set the destructor inside capsule to destructor. - - Return 0 on success. Return nonzero and set an exception on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) -def PyCapsule_SetName(space, capsule, name): - """Set the name inside capsule to name. If non-NULL, the name must - outlive the capsule. If the previous name stored in the capsule was not - NULL, no attempt is made to free it. - - Return 0 on success. Return nonzero and set an exception on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, {void*}], rffi.INT_real) -def PyCapsule_SetPointer(space, capsule, pointer): - """Set the void pointer inside capsule to pointer. The pointer may not be - NULL. - - Return 0 on success. Return nonzero and set an exception on failure.""" - raise NotImplementedError - - at cpython_api([{ob}], rffi.INT_real) -def PyCell_Check(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyCell_Check(space, cell): """Return true if ob is a cell object; ob must not be NULL.""" raise NotImplementedError @@ -795,7 +650,7 @@ non-NULL and a cell object.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyCell_Set(space, cell, value): """Set the contents of the cell object cell to value. This releases the reference to any current content of the cell. value may be NULL. cell @@ -810,59 +665,59 @@ be a cell object.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyClass_Check(space, o): """Return true if the object o is a class object, including instances of types derived from the standard class object. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyClass_IsSubclass(space, klass, base): """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyInstance_Check(space, obj): """Return true if obj is an instance.""" raise NotImplementedError @cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, class, arg, kw): +def PyInstance_New(space, cls, arg, kw): """Create a new instance of a specific class. The parameters arg and kw are used as the positional and keyword parameters to the object's constructor.""" raise NotImplementedError @cpython_api([PyObject, PyObject], PyObject) -def PyInstance_NewRaw(space, class, dict): +def PyInstance_NewRaw(space, cls, dict): """Create a new instance of a specific class without calling its constructor. class is the class of new object. The dict parameter will be used as the object's __dict__; if NULL, a new dictionary will be created for the instance.""" raise NotImplementedError - at cpython_api([PyObject], {void*}) + at cpython_api([PyObject], rffi.VOIDP_real, error=lltype.nullptr(rffi.VOIDP.TO)) def PyCObject_GetDesc(space, self): """Return the description void * that the PyCObject self was created with.""" raise NotImplementedError - at cpython_api([PyObject, {void*}], rffi.INT_real) + at cpython_api([PyObject, rffi.VOIDP_real], rffi.INT_real, error=0) def PyCObject_SetVoidPtr(space, self, cobj): """Set the void pointer inside self to cobj. The PyCObject must not have an associated destructor. Return true on success, false on failure.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCode_Check(space, co): """Return true if co is a code object""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=-1) def PyCode_GetNumFree(space, co): """Return the number of free variables in co.""" raise NotImplementedError - at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, rffi.INT_real, PyObject], {PyCodeObject*}) + at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, PyObject, rffi.INT_real, PyObject], PyObject) def PyCode_New(space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, freevars, cellvars, filename, name, firstlineno, lnotab): """Return a new code object. If you need a dummy code object to create a frame, use PyCode_NewEmpty() instead. Calling @@ -870,85 +725,55 @@ version since the definition of the bytecode changes often.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real], rffi.INT_real) + at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyCode_NewEmpty(space, filename, funcname, firstlineno): """Return a new empty code object with the specified filename, function name, and first line number. It is illegal to exec or eval() the resulting code object.""" raise NotImplementedError - at cpython_api([{Py_complex}, {Py_complex}], {Py_complex}) + at cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_sum(space, left, right): """Return the sum of two complex numbers, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([{Py_complex}, {Py_complex}], {Py_complex}) + at cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_diff(space, left, right): """Return the difference between two complex numbers, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([{Py_complex}], {Py_complex}) + at cpython_api([Py_complex], Py_complex) def _Py_c_neg(space, complex): """Return the negation of the complex number complex, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([{Py_complex}, {Py_complex}], {Py_complex}) + at cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_prod(space, left, right): """Return the product of two complex numbers, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([{Py_complex}, {Py_complex}], {Py_complex}) + at cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_quot(space, dividend, divisor): """Return the quotient of two complex numbers, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([{Py_complex}, {Py_complex}], {Py_complex}) + at cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_pow(space, num, exp): """Return the exponentiation of num by exp, using the C Py_complex representation.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyComplex_Check(space, p): - """Return true if its argument is a PyComplexObject or a subtype of - PyComplexObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyComplex_CheckExact(space, p): - """Return true if its argument is a PyComplexObject, but not a subtype of - PyComplexObject. - """ - raise NotImplementedError - - at cpython_api([{Py_complex}], PyObject) + at cpython_api([Py_complex], PyObject) def PyComplex_FromCComplex(space, v): """Create a new Python complex number object from a C Py_complex value.""" raise NotImplementedError - at cpython_api([{double}, {double}], PyObject) -def PyComplex_FromDoubles(space, real, imag): - """Return a new PyComplexObject object from real and imag.""" - raise NotImplementedError - - at cpython_api([PyObject], {double}) -def PyComplex_RealAsDouble(space, op): - """Return the real part of op as a C double.""" - raise NotImplementedError - - at cpython_api([PyObject], {double}) -def PyComplex_ImagAsDouble(space, op): - """Return the imaginary part of op as a C double.""" - raise NotImplementedError - - at cpython_api([PyObject], {Py_complex}) + at cpython_api([PyObject], Py_complex) def PyComplex_AsCComplex(space, op): """Return the Py_complex value of the complex number op. @@ -957,20 +782,20 @@ number object.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {size_t}, rffi.CCHARP, ...], rffi.INT_real) + at cpython_api([rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, ], rffi.INT_real, error=CANNOT_FAIL) def PyOS_snprintf(space, str, size, format, ): """Output not more than size bytes to str according to the format string format and the extra arguments. See the Unix man page snprintf(2).""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {size_t}, rffi.CCHARP, {va_list}], rffi.INT_real) + at cpython_api([rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, va_list], rffi.INT_real, error=CANNOT_FAIL) def PyOS_vsnprintf(space, str, size, format, va): """Output not more than size bytes to str according to the format string format and the variable argument list va. Unix man page vsnprintf(2).""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {char**}, PyObject], {double}) + at cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) def PyOS_string_to_double(space, s, endptr, overflow_exception): """Convert a string s to a double, raising a Python exception on failure. The set of accepted strings corresponds to @@ -1003,7 +828,7 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, {char**}], {double}) + at cpython_api([rffi.CCHARP, rffi.CCHARPP], rffi.DOUBLE, error=CANNOT_FAIL) def PyOS_ascii_strtod(space, nptr, endptr): """Convert a string to a double. This function behaves like the Standard C function strtod() does in the C locale. It does this without changing the @@ -1019,7 +844,7 @@ Use PyOS_string_to_double() instead.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {size_t}, rffi.CCHARP, {double}], rffi.CCHARP) + at cpython_api([rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, rffi.DOUBLE], rffi.CCHARP, error=lltype.nullptr(rffi.CCHARP.TO)) def PyOS_ascii_formatd(space, buffer, buf_len, format, d): """Convert a double to a string using the '.' as the decimal separator. format is a printf()-style format string specifying the @@ -1035,7 +860,7 @@ instead.""" raise NotImplementedError - at cpython_api([{double}, lltype.Char, rffi.INT_real, rffi.INT_real, {int*}], rffi.CCHARP) + at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP, error=lltype.nullptr(rffi.CCHARP.TO)) def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): """Convert a double val to a string using supplied format_code, precision, and flags. @@ -1068,7 +893,7 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP], {double}) + at cpython_api([rffi.CCHARP], rffi.DOUBLE, error=CANNOT_FAIL) def PyOS_ascii_atof(space, nptr): """Convert a string to a double in a locale-independent way. @@ -1079,84 +904,84 @@ Use PyOS_string_to_double() instead.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) + at cpython_api([rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, error=CANNOT_FAIL) def PyOS_stricmp(space, s1, s2): """Case insensitive comparison of strings. The function works almost identically to strcmp() except that it ignores the case. """ raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, {Py_ssize_t }], rffi.CCHARP) + at cpython_api([rffi.CCHARP, rffi.CCHARP, Py_ssize_t], rffi.CCHARP, error=CANNOT_FAIL) def PyOS_strnicmp(space, s1, s2, size): """Case insensitive comparison of strings. The function works almost identically to strncmp() except that it ignores the case. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDate_Check(space, ob): """Return true if ob is of type PyDateTime_DateType or a subtype of PyDateTime_DateType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDate_CheckExact(space, ob): """Return true if ob is of type PyDateTime_DateType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_Check(space, ob): """Return true if ob is of type PyDateTime_DateTimeType or a subtype of PyDateTime_DateTimeType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_CheckExact(space, ob): """Return true if ob is of type PyDateTime_DateTimeType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyTime_Check(space, ob): """Return true if ob is of type PyDateTime_TimeType or a subtype of PyDateTime_TimeType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyTime_CheckExact(space, ob): """Return true if ob is of type PyDateTime_TimeType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDelta_Check(space, ob): """Return true if ob is of type PyDateTime_DeltaType or a subtype of PyDateTime_DeltaType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDelta_CheckExact(space, ob): """Return true if ob is of type PyDateTime_DeltaType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyTZInfo_Check(space, ob): """Return true if ob is of type PyDateTime_TZInfoType or a subtype of PyDateTime_TZInfoType. ob must not be NULL. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyTZInfo_CheckExact(space, ob): """Return true if ob is of type PyDateTime_TZInfoType. ob must not be NULL. @@ -1176,13 +1001,6 @@ """ raise NotImplementedError - at cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject) -def PyTime_FromTime(space, hour, minute, second, usecond): - """Return a datetime.time object with the specified hour, minute, second and - microsecond. - """ - raise NotImplementedError - @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real], PyObject) def PyDelta_FromDSU(space, days, seconds, useconds): """Return a datetime.timedelta object representing the given number of days, @@ -1192,67 +1010,67 @@ """ raise NotImplementedError - at cpython_api([{PyDateTime_Date*}], rffi.INT_real) + at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_GET_YEAR(space, o): """Return the year, as a positive int. """ raise NotImplementedError - at cpython_api([{PyDateTime_Date*}], rffi.INT_real) + at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_GET_MONTH(space, o): """Return the month, as an int from 1 through 12. """ raise NotImplementedError - at cpython_api([{PyDateTime_Date*}], rffi.INT_real) + at cpython_api([PyDateTime_Date], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_GET_DAY(space, o): """Return the day, as an int from 1 through 31. """ raise NotImplementedError - at cpython_api([{PyDateTime_DateTime*}], rffi.INT_real) + at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_DATE_GET_HOUR(space, o): """Return the hour, as an int from 0 through 23. """ raise NotImplementedError - at cpython_api([{PyDateTime_DateTime*}], rffi.INT_real) + at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_DATE_GET_MINUTE(space, o): """Return the minute, as an int from 0 through 59. """ raise NotImplementedError - at cpython_api([{PyDateTime_DateTime*}], rffi.INT_real) + at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_DATE_GET_SECOND(space, o): """Return the second, as an int from 0 through 59. """ raise NotImplementedError - at cpython_api([{PyDateTime_DateTime*}], rffi.INT_real) + at cpython_api([PyDateTime_DateTime], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_DATE_GET_MICROSECOND(space, o): """Return the microsecond, as an int from 0 through 999999. """ raise NotImplementedError - at cpython_api([{PyDateTime_Time*}], rffi.INT_real) + at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_TIME_GET_HOUR(space, o): """Return the hour, as an int from 0 through 23. """ raise NotImplementedError - at cpython_api([{PyDateTime_Time*}], rffi.INT_real) + at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_TIME_GET_MINUTE(space, o): """Return the minute, as an int from 0 through 59. """ raise NotImplementedError - at cpython_api([{PyDateTime_Time*}], rffi.INT_real) + at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_TIME_GET_SECOND(space, o): """Return the second, as an int from 0 through 59. """ raise NotImplementedError - at cpython_api([{PyDateTime_Time*}], rffi.INT_real) + at cpython_api([PyDateTime_Time], rffi.INT_real, error=CANNOT_FAIL) def PyDateTime_TIME_GET_MICROSECOND(space, o): """Return the microsecond, as an int from 0 through 999999. """ @@ -1272,19 +1090,19 @@ """ raise NotImplementedError - at cpython_api([PyTypeObjectPtr, {struct PyGetSetDef*}], PyObject) + at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) def PyDescr_NewGetSet(space, type, getset): raise NotImplementedError - at cpython_api([PyTypeObjectPtr, {struct PyMemberDef*}], PyObject) + at cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError - at cpython_api([PyTypeObjectPtr, {struct PyMethodDef*}], PyObject) + at cpython_api([PyTypeObjectPtr, PyMethodDef], PyObject) def PyDescr_NewMethod(space, type, meth): raise NotImplementedError - at cpython_api([PyTypeObjectPtr, {struct wrapperbase*}, {void*}], PyObject) + at cpython_api([PyTypeObjectPtr, wrapperbase, rffi.VOIDP_real], PyObject) def PyDescr_NewWrapper(space, type, wrapper, wrapped): raise NotImplementedError @@ -1292,7 +1110,7 @@ def PyDescr_NewClassMethod(space, type, method): raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyDescr_IsData(space, descr): """Return true if the descriptor objects descr describes a data attribute, or false if it describes a method. descr must be a descriptor object; there is @@ -1301,7 +1119,7 @@ raise NotImplementedError @cpython_api([PyObject, PyObject], PyObject) -def PyWrapper_New(space, , ): +def PyWrapper_New(space, a, b): raise NotImplementedError @cpython_api([PyObject], PyObject) @@ -1317,7 +1135,7 @@ """Empty an existing dictionary of all key-value pairs.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_Contains(space, p, key): """Determine if dictionary p contains key. If an item in p is matches key, return 1, otherwise return 0. On error, return -1. @@ -1331,14 +1149,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) -def PyDict_DelItem(space, p, key): - """Remove the entry in dictionary p with key key. key must be hashable; - if it isn't, TypeError is raised. Return 0 on success or -1 - on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyDict_DelItemString(space, p, key): """Remove the entry in dictionary p which has a key specified by the string key. Return 0 on success or -1 on failure.""" @@ -1362,67 +1173,7 @@ dictionary p, as in the dictionary method dict.values().""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyDict_Size(space, p): - """ - - - - Return the number of items in the dictionary. This is equivalent to - len(p) on a dictionary. - - This function returned an int type. This might require changes - in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real) -def PyDict_Next(space, p, ppos, pkey, pvalue): - """Iterate over all key-value pairs in the dictionary p. The - Py_ssize_t referred to by ppos must be initialized to 0 - prior to the first call to this function to start the iteration; the - function returns true for each pair in the dictionary, and false once all - pairs have been reported. The parameters pkey and pvalue should either - point to PyObject* variables that will be filled in with each key - and value, respectively, or may be NULL. Any references returned through - them are borrowed. ppos should not be altered during iteration. Its - value represents offsets within the internal dictionary structure, and - since the structure is sparse, the offsets are not consecutive. - - For example: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - /* do something interesting with the values... */ - ... - } - - The dictionary p should not be mutated during iteration. It is safe - (since Python 2.1) to modify the values of the keys as you iterate over the - dictionary, but only so long as the set of keys does not change. For - example: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - int i = PyInt_AS_LONG(value) + 1; - PyObject *o = PyInt_FromLong(i); - if (o == NULL) - return -1; - if (PyDict_SetItem(self->dict, key, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); - } - - This function used an int * type for ppos. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) def PyDict_Merge(space, a, b, override): """Iterate over mapping object b adding key-value pairs to dictionary a. b may be a dictionary, or any object supporting PyMapping_Keys() @@ -1433,14 +1184,14 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_Update(space, a, b): """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in Python. Return 0 on success or -1 if an exception was raised. """ raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) def PyDict_MergeFromSeq2(space, a, seq2, override): """Update or merge into dictionary a, from the key-value pairs in seq2. seq2 must be an iterable object producing iterable objects of length 2, @@ -1467,13 +1218,8 @@ type, value and traceback of the printed exception, respectively.""" raise NotImplementedError - at cpython_api([], lltype.Void) -def PyErr_Print(space, ): - """Alias for PyErr_PrintEx(1).""" - raise NotImplementedError - - at cpython_api([{PyObject**exc}, {PyObject**val}, {PyObject**tb}], lltype.Void) -def PyErr_NormalizeException(space, , , ): + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc, val, tb): """Under certain circumstances, the values returned by PyErr_Fetch() below can be "unnormalized", meaning that *exc is a class object but *val is not an instance of the same class. This function can be used to instantiate @@ -1481,7 +1227,7 @@ The delayed normalization is implemented to improve performance.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ...], PyObject) + at cpython_api([PyObject, rffi.CCHARP, ], PyObject) def PyErr_Format(space, exception, format, ): """This function sets the error indicator and returns NULL. exception should be a Python exception (class, not an instance). format should be a string, @@ -1629,13 +1375,6 @@ Return value: always NULL.""" raise NotImplementedError - at cpython_api([], rffi.INT_real) -def PyErr_BadArgument(space, ): - """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where - message indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyErr_SetFromErrnoWithFilename(space, type, filename): """Similar to PyErr_SetFromErrno(), with the additional behavior that if @@ -1684,54 +1423,7 @@ Return value: always NULL.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, rffi.INT_real], rffi.INT_real) -def PyErr_WarnEx(space, category, message, stacklevel): - """Issue a warning message. The category argument is a warning category (see - below) or NULL; the message argument is a message string. stacklevel is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A stacklevel of 1 - is the function calling PyErr_WarnEx(), 2 is the function above that, - and so forth. - - This function normally prints a warning message to sys.stderr; however, it is - also possible that the user has specified that warnings are to be turned into - errors, and in that case this will raise an exception. It is also possible that - the function raises an exception because of a problem with the warning machinery - (the implementation imports the warnings module to do the heavy lifting). - The return value is 0 if no exception is raised, or -1 if an exception - is raised. (It is not possible to determine whether a warning message is - actually printed, nor what the reason is for the exception; this is - intentional.) If an exception is raised, the caller should do its normal - exception handling (for example, Py_DECREF() owned references and return - an error value). - - Warning categories must be subclasses of Warning; the default warning - category is RuntimeWarning. The standard Python warning categories are - available as global variables whose names are PyExc_ followed by the Python - exception name. These have the type PyObject*; they are all class - objects. Their names are PyExc_Warning, PyExc_UserWarning, - PyExc_UnicodeWarning, PyExc_DeprecationWarning, - PyExc_SyntaxWarning, PyExc_RuntimeWarning, and - PyExc_FutureWarning. PyExc_Warning is a subclass of - PyExc_Exception; the other warning categories are subclasses of - PyExc_Warning. - - For information about warning control, see the documentation for the - warnings module and the -W option in the command line - documentation. There is no C API for warning control.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) -def PyErr_Warn(space, category, message): - """Issue a warning message. The category argument is a warning category (see - below) or NULL; the message argument is a message string. The warning will - appear to be issued from the function calling PyErr_Warn(), equivalent to - calling PyErr_WarnEx() with a stacklevel of 1. - - Deprecated; use PyErr_WarnEx() instead.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.CCHARP, PyObject], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.CCHARP, PyObject], rffi.INT_real, error=-1) def PyErr_WarnExplicit(space, category, message, filename, lineno, module, registry): """Issue a warning message with explicit control over all warning attributes. This is a straightforward wrapper around the Python function @@ -1740,7 +1432,7 @@ described there.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real], rffi.INT_real) + at cpython_api([rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) def PyErr_WarnPy3k(space, message, stacklevel): """Issue a DeprecationWarning with the given message and stacklevel if the Py_Py3kWarningFlag flag is enabled. @@ -1762,7 +1454,7 @@ % thread.interrupt_main() (used from IDLE), so it's still needed.""" raise NotImplementedError - at cpython_api([rffi.INT_real], rffi.INT_real) + at cpython_api([rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL) def PySignal_SetWakeupFd(space, fd): """This utility function specifies a file descriptor to which a '\0' byte will be written whenever a signal is received. It returns the previous such file @@ -1807,7 +1499,7 @@ the warning message.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], rffi.INT_real) + at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1) def Py_EnterRecursiveCall(space, where): """Marks a point where a recursive C-level call is about to be performed. @@ -1830,7 +1522,7 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyFile_Check(space, p): """Return true if its argument is a PyFileObject or a subtype of PyFileObject. @@ -1838,7 +1530,7 @@ Allowed subtypes to be accepted.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyFile_CheckExact(space, p): """Return true if its argument is a PyFileObject, but not a subtype of PyFileObject. @@ -1856,23 +1548,14 @@ semantics as the standard C routine fopen(). On failure, return NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) -def PyFile_FromFile(space, fp, name, mode, (*close)(FILE*)): + at cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) +def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file pointer, fp. The function close will be called when the file should be closed. Return NULL on failure.""" raise NotImplementedError - at cpython_api([PyObject], {FILE*}) -def PyFile_AsFile(space, p): - """Return the file object associated with p as a FILE*. - - If the caller will ever use the returned FILE* object while - the GIL is released it must also call the PyFile_IncUseCount() and - PyFile_DecUseCount() functions described below as appropriate.""" - raise NotImplementedError - - at cpython_api([{PyFileObject*}], lltype.Void) + at cpython_api([PyFileObject], lltype.Void) def PyFile_IncUseCount(space, p): """Increments the PyFileObject's internal use count to indicate that the underlying FILE* is being used. @@ -1888,7 +1571,7 @@ """ raise NotImplementedError - at cpython_api([{PyFileObject*}], lltype.Void) + at cpython_api([PyFileObject], lltype.Void) def PyFile_DecUseCount(space, p): """Decrements the PyFileObject's internal unlocked_count member to indicate that the caller is done with its own use of the FILE*. @@ -1919,36 +1602,30 @@ """Return the name of the file specified by p as a string object.""" raise NotImplementedError - at cpython_api([{PyFileObject*}, rffi.INT_real], lltype.Void) + at cpython_api([PyFileObject, rffi.INT_real], lltype.Void) def PyFile_SetBufSize(space, p, n): """ - - - Available on systems with setvbuf() only. This should only be called immediately after file object creation.""" raise NotImplementedError - at cpython_api([{PyFileObject*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyFileObject, rffi.CCHARP], rffi.INT_real, error=0) def PyFile_SetEncoding(space, p, enc): """Set the file's encoding for Unicode output to enc. Return 1 on success and 0 on failure. """ raise NotImplementedError - at cpython_api([{PyFileObject*}, rffi.CCHARP, {*errors}], rffi.INT_real) -def PyFile_SetEncodingAndErrors(space, p, enc, ): + at cpython_api([PyFileObject, rffi.CCHARP, rffi.CCHARP], rffi.INT_real, error=0) +def PyFile_SetEncodingAndErrors(space, p, enc, errors): """Set the file's encoding for Unicode output to enc, and its error mode to err. Return 1 on success and 0 on failure. """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL) def PyFile_SoftSpace(space, p, newflag): """ - - - This function exists for internal use by the interpreter. Set the softspace attribute of p to newflag and return the previous value. p does not have to be a file object for this function to work properly; any @@ -1959,40 +1636,22 @@ but doing so should not be needed.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) def PyFile_WriteObject(space, obj, p, flags): """ - - - Write object obj to file object p. The only supported flag for flags is Py_PRINT_RAW; if given, the str() of the object is written instead of the repr(). Return 0 on success or -1 on failure; the appropriate exception will be set.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, PyObject], rffi.INT_real) + at cpython_api([rffi.CCHARP, PyObject], rffi.INT_real, error=-1) def PyFile_WriteString(space, s, p): """Write string s to file object p. Return 0 on success or -1 on failure; the appropriate exception will be set.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyFloat_Check(space, p): - """Return true if its argument is a PyFloatObject or a subtype of - PyFloatObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyFloat_CheckExact(space, p): - """Return true if its argument is a PyFloatObject, but not a subtype of - PyFloatObject. - """ - raise NotImplementedError - - at cpython_api([PyObject, {char**}], PyObject) + at cpython_api([PyObject, rffi.CCHARPP], PyObject) def PyFloat_FromString(space, str, pend): """Create a PyFloatObject object based on the string value in str, or NULL on failure. The pend argument is ignored. It remains only for @@ -2000,33 +1659,33 @@ raise NotImplementedError @cpython_api([rffi.VOIDP_real], PyObject) -def PyFloat_GetInfo(space, ): +def PyFloat_GetInfo(space, info): """Return a structseq instance which contains information about the precision, minimum and maximum values of a float. It's a thin wrapper around the header file float.h. """ raise NotImplementedError - at cpython_api([], {double}) + at cpython_api([], rffi.DOUBLE, error=CANNOT_FAIL) def PyFloat_GetMax(space, ): """Return the maximum representable finite float DBL_MAX as C double. """ raise NotImplementedError - at cpython_api([], {double}) + at cpython_api([], rffi.DOUBLE, error=CANNOT_FAIL) def PyFloat_GetMin(space, ): """Return the minimum normalized positive float DBL_MIN as C double. """ raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyFloat_ClearFreeList(space, ): """Clear the float free list. Return the number of items that could not be freed. """ raise NotImplementedError - at cpython_api([rffi.CCHARP, {PyFloatObject*}], lltype.Void) + at cpython_api([rffi.CCHARP, PyObject], lltype.Void) def PyFloat_AsString(space, buf, v): """Convert the argument v to a string, using the same rules as str(). The length of buf should be at least 100. @@ -2037,7 +1696,7 @@ Use PyObject_Str() or PyOS_double_to_string() instead.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {PyFloatObject*}], lltype.Void) + at cpython_api([rffi.CCHARP, PyObject], lltype.Void) def PyFloat_AsReprString(space, buf, v): """Same as PyFloat_AsString, except uses the same rules as repr(). The length of buf should be at least 100. @@ -2048,7 +1707,7 @@ Use PyObject_Repr() or PyOS_double_to_string() instead.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyFunction_Check(space, o): """Return true if o is a function object (has type PyFunction_Type). The parameter must not be NULL.""" @@ -2086,7 +1745,7 @@ tuple of arguments or NULL.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyFunction_SetDefaults(space, op, defaults): """Set the argument default values for the function object op. defaults must be Py_None or a tuple. @@ -2100,7 +1759,7 @@ or a tuple of cell objects.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyFunction_SetClosure(space, op, closure): """Set the closure associated with the function object op. closure must be Py_None or a tuple of cell objects. @@ -2108,14 +1767,8 @@ Raises SystemError and returns -1 on failure.""" raise NotImplementedError - at cpython_api([{TYPE}, PyTypeObjectPtr], {TYPE*}) -def PyObject_GC_New(space, , type): - """Analogous to PyObject_New() but for container objects with the - Py_TPFLAGS_HAVE_GC flag set.""" - raise NotImplementedError - - at cpython_api([{TYPE}, PyTypeObjectPtr, Py_ssize_t], {TYPE*}) -def PyObject_GC_NewVar(space, , type, size): + at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) +def _PyObject_GC_NewVar(space, type, size): """Analogous to PyObject_NewVar() but for container objects with the Py_TPFLAGS_HAVE_GC flag set. @@ -2123,8 +1776,8 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{TYPE}, PyObject, Py_ssize_t], {TYPE*}) -def PyObject_GC_Resize(space, , op, newsize): + at cpython_api([PyObject, Py_ssize_t], PyObject) +def _PyObject_GC_Resize(space, op, newsize): """Resize an object allocated by PyObject_NewVar(). Returns the resized object or NULL on failure. @@ -2133,35 +1786,11 @@ raise NotImplementedError @cpython_api([PyObject], lltype.Void) -def PyObject_GC_Track(space, op): - """Adds the object op to the set of container objects tracked by the - collector. The collector can run at unexpected times so objects must be - valid while being tracked. This should be called once all the fields - followed by the tp_traverse handler become valid, usually near the - end of the constructor.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) def _PyObject_GC_TRACK(space, op): """A macro version of PyObject_GC_Track(). It should not be used for extension modules.""" raise NotImplementedError - at cpython_api([{void*}], lltype.Void) -def PyObject_GC_Del(space, op): - """Releases memory allocated to an object using PyObject_GC_New() or - PyObject_GC_NewVar().""" - raise NotImplementedError - - at cpython_api([{void*}], lltype.Void) -def PyObject_GC_UnTrack(space, op): - """Remove the object op from the set of container objects tracked by the - collector. Note that PyObject_GC_Track() can be called again on - this object to add it back to the set of tracked objects. The deallocator - (tp_dealloc handler) should call this for the object before any of - the fields used by the tp_traverse handler become invalid.""" - raise NotImplementedError - @cpython_api([PyObject], lltype.Void) def _PyObject_GC_UNTRACK(space, op): """A macro version of PyObject_GC_UnTrack(). It should not be used for @@ -2184,18 +1813,18 @@ """ raise NotImplementedError - at cpython_api([{ob}], rffi.INT_real) -def PyGen_Check(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyGen_Check(space, gen): """Return true if ob is a generator object; ob must not be NULL.""" raise NotImplementedError - at cpython_api([{ob}], rffi.INT_real) -def PyGen_CheckExact(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyGen_CheckExact(space, gen): """Return true if ob's type is PyGen_Type is a generator object; ob must not be NULL.""" raise NotImplementedError - at cpython_api([{PyFrameObject*}], PyObject) + at cpython_api([PyFrameObject], PyObject) def PyGen_New(space, frame): """Create and return a new generator object based on the frame object. A reference to frame is stolen by this function. The parameter must not be @@ -2203,31 +1832,6 @@ raise NotImplementedError @cpython_api([rffi.CCHARP], PyObject) -def PyImport_ImportModule(space, name): - """ - - - - This is a simplified interface to PyImport_ImportModuleEx() below, - leaving the globals and locals arguments set to NULL and level set - to 0. When the name - argument contains a dot (when it specifies a submodule of a package), the - fromlist argument is set to the list ['*'] so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when name in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's __all__ variable are loaded.) Return a new reference to the - imported module, or NULL with an exception set on failure. Before Python 2.4, - the module may still be created in the failure case --- examine sys.modules - to find out. Starting with Python 2.4, a failing import of a module no longer - leaves the module in sys.modules. - - Failing imports remove incomplete module objects. - - Always uses absolute imports.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP], PyObject) def PyImport_ImportModuleNoBlock(space, name): """This version of PyImport_ImportModule() does not block. It's intended to be used in C functions that import other modules to execute a function. @@ -2275,20 +1879,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject) -def PyImport_Import(space, name): - """ - - - - This is a higher-level interface that calls the current "import hook function". - It invokes the __import__() function from the __builtins__ of the - current globals. This means that the import is done using whatever import hooks - are installed in the current environment, e.g. by rexec or ihooks. - - Always uses absolute imports.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) def PyImport_ReloadModule(space, m): """ @@ -2339,7 +1929,7 @@ name is removed from sys.modules in error cases.""" raise NotImplementedError - at cpython_api([], lltype.Signed) + at cpython_api([], lltype.Signed, error=CANNOT_FAIL) def PyImport_GetMagicNumber(space, ): """Return the magic number for Python bytecode files (a.k.a. .pyc and .pyo files). The magic number should be present in the first four bytes @@ -2379,17 +1969,7 @@ """Finalize the import mechanism. For internal use only.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP], PyObject) -def _PyImport_FindExtension(space, , ): - """For internal use only.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, rffi.CCHARP], PyObject) -def _PyImport_FixupExtension(space, , ): - """For internal use only.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP], rffi.INT_real) + at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1) def PyImport_ImportFrozenModule(space, name): """Load a frozen module named name. Return 1 for success, 0 if the module is not found, and -1 with an exception set if the initialization @@ -2398,8 +1978,8 @@ reload the module if it was already imported.)""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.VOIDP_real], rffi.INT_real) -def PyImport_AppendInittab(space, name, (*initfunc)(void)): + at cpython_api([rffi.CCHARP, rffi.VOIDP_real], rffi.INT_real, error=-1) +def PyImport_AppendInittab(space, name, initfunc): """Add a single module to the existing table of built-in modules. This is a convenience wrapper around PyImport_ExtendInittab(), returning -1 if the table could not be extended. The new module can be imported by the name @@ -2408,7 +1988,7 @@ Py_Initialize().""" raise NotImplementedError - at cpython_api([{struct _inittab*}], rffi.INT_real) + at cpython_api([_inittab], rffi.INT_real, error=-1) def PyImport_ExtendInittab(space, newtab): """Add a collection of modules to the table of built-in modules. The newtab array must end with a sentinel entry which contains NULL for the name @@ -2474,7 +2054,7 @@ Py_Finalize() more than once.""" raise NotImplementedError - at cpython_api([], {PyThreadState*}) + at cpython_api([], PyThreadState) def Py_NewInterpreter(space, ): """ @@ -2540,7 +2120,7 @@ things may work, but confusing behavior will always be near.""" raise NotImplementedError - at cpython_api([{PyThreadState*}], lltype.Void) + at cpython_api([PyThreadState], lltype.Void) def Py_EndInterpreter(space, tstate): """ @@ -2572,7 +2152,7 @@ interpreter will change the contents of this storage.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetProgramName(space, ): """ @@ -2583,7 +2163,7 @@ value.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with @@ -2596,7 +2176,7 @@ It is only useful on Unix. See also the next function.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetExecPrefix(space, ): """Return the exec-prefix for installed platform-dependent files. This is derived through a number of complicated rules from the program name set with @@ -2631,7 +2211,7 @@ platform.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetProgramFullPath(space, ): """ @@ -2644,7 +2224,7 @@ to Python code as sys.executable.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPath(space, ): """ @@ -2663,12 +2243,12 @@ XXX should give the exact rules""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetVersion(space, ): """Return the version of this Python interpreter. This is a string that looks something like - "1.5 (#67, Dec 31 1997, 22:34:28) [GCC 2.7.2.2]" + "1.5 (\#67, Dec 31 1997, 22:34:28) [GCC 2.7.2.2]" @@ -2680,7 +2260,7 @@ modify its value. The value is available to Python code as sys.version.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPlatform(space, ): """ @@ -2695,7 +2275,7 @@ to Python code as sys.platform.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetCopyright(space, ): """Return the official copyright string for the current Python version, for example @@ -2709,7 +2289,7 @@ value. The value is available to Python code as sys.copyright.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetCompiler(space, ): """Return an indication of the compiler used to build the current Python version, in square brackets, for example: @@ -2725,12 +2305,12 @@ sys.version.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetBuildInfo(space, ): """Return information about the sequence number and build date and time of the current Python interpreter instance, for example - "#67, Aug 1 1997, 22:34:28" + "\#67, Aug 1 1997, 22:34:28" @@ -2741,7 +2321,7 @@ sys.version.""" raise NotImplementedError - at cpython_api([rffi.INT_real, {char**}], lltype.Void) + at cpython_api([rffi.INT_real, rffi.CCHARPP], lltype.Void) def PySys_SetArgv(space, argc, argv): """ @@ -2774,7 +2354,7 @@ this storage.""" raise NotImplementedError - at cpython_api([], rffi.CCHARP) + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPythonHome(space, ): """Return the default "home", that is, the value set by a previous call to Py_SetPythonHome(), or the value of the PYTHONHOME @@ -2794,24 +2374,6 @@ This function is not available when thread support is disabled at compile time.""" raise NotImplementedError - at cpython_api([], {PyThreadState*}) -def PyEval_SaveThread(space, ): - """Release the global interpreter lock (if it has been created and thread - support is enabled) and reset the thread state to NULL, returning the - previous thread state (which is not NULL). If the lock has been created, - the current thread must have acquired it. (This function is available even - when thread support is disabled at compile time.)""" - raise NotImplementedError - - at cpython_api([{PyThreadState*}], lltype.Void) -def PyEval_RestoreThread(space, tstate): - """Acquire the global interpreter lock (if it has been created and thread - support is enabled) and set the thread state to tstate, which must not be - NULL. If the lock has been created, the current thread must not have - acquired it, otherwise deadlock ensues. (This function is available even - when thread support is disabled at compile time.)""" - raise NotImplementedError - @cpython_api([], lltype.Void) def PyEval_ReInitThreads(space, ): """This function is called from PyOS_AfterFork() to ensure that newly @@ -2819,27 +2381,27 @@ are not running in the child process.""" raise NotImplementedError - at cpython_api([], {PyInterpreterState*}) + at cpython_api([], PyInterpreterState) def PyInterpreterState_New(space, ): """Create a new interpreter state object. The global interpreter lock need not be held, but may be held if it is necessary to serialize calls to this function.""" raise NotImplementedError - at cpython_api([{PyInterpreterState*}], lltype.Void) + at cpython_api([PyInterpreterState], lltype.Void) def PyInterpreterState_Clear(space, interp): """Reset all information in an interpreter state object. The global interpreter lock must be held.""" raise NotImplementedError - at cpython_api([{PyInterpreterState*}], lltype.Void) + at cpython_api([PyInterpreterState], lltype.Void) def PyInterpreterState_Delete(space, interp): """Destroy an interpreter state object. The global interpreter lock need not be held. The interpreter state must have been reset with a previous call to PyInterpreterState_Clear().""" raise NotImplementedError - at cpython_api([], {PyThreadState*}) + at cpython_api([], PyThreadState) def PyThreadState_Get(space, ): """Return the current thread state. The global interpreter lock must be held. When the current thread state is NULL, this issues a fatal error (so that @@ -2858,7 +2420,7 @@ meant that an exception was raised.""" raise NotImplementedError - at cpython_api([lltype.Signed, PyObject], rffi.INT_real) + at cpython_api([lltype.Signed, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyThreadState_SetAsyncExc(space, id, exc): """Asynchronously raise an exception in a thread. The id argument is the thread id of the target thread; exc is the exception object to be raised. This @@ -2870,8 +2432,8 @@ """ raise NotImplementedError - at cpython_api([{int (*func)(void*}, {void*}], lltype.Void) -def Py_AddPendingCall(space, , arg)): + at cpython_api([rffi.VOIDP_real, rffi.VOIDP_real], lltype.Void) +def Py_AddPendingCall(space, func, arg): """ @@ -2896,7 +2458,7 @@ """ raise NotImplementedError - at cpython_api([{Py_tracefunc}, PyObject], lltype.Void) + at cpython_api([PyObject, PyObject], lltype.Void) def PyEval_SetProfile(space, func, obj): """Set the profiler function to func. The obj parameter is passed to the function as its first parameter, and may be any Python object, or NULL. If @@ -2906,7 +2468,7 @@ events.""" raise NotImplementedError - at cpython_api([{Py_tracefunc}, PyObject], lltype.Void) + at cpython_api([PyObject, PyObject], lltype.Void) def PyEval_SetTrace(space, func, obj): """Set the tracing function to func. This is similar to PyEval_SetProfile(), except the tracing function does receive line-number @@ -2981,34 +2543,34 @@ defined.""" raise NotImplementedError - at cpython_api([], {PyInterpreterState*}) + at cpython_api([], PyInterpreterState) def PyInterpreterState_Head(space, ): """Return the interpreter state object at the head of the list of all such objects. """ raise NotImplementedError - at cpython_api([{PyInterpreterState*}], {PyInterpreterState*}) + at cpython_api([PyInterpreterState], PyInterpreterState) def PyInterpreterState_Next(space, interp): """Return the next interpreter state object after interp from the list of all such objects. """ raise NotImplementedError - at cpython_api([{PyInterpreterState*}], {PyThreadState* }) + at cpython_api([PyInterpreterState], PyThreadState) def PyInterpreterState_ThreadHead(space, interp): """Return the a pointer to the first PyThreadState object in the list of threads associated with the interpreter interp. """ raise NotImplementedError - at cpython_api([{PyThreadState*}], {PyThreadState*}) + at cpython_api([PyThreadState], PyThreadState) def PyThreadState_Next(space, tstate): """Return the next thread state object after tstate from the list of all such objects belonging to the same PyInterpreterState object. """ raise NotImplementedError - at cpython_api([rffi.CCHARP, {char**}, rffi.INT_real], PyObject) + at cpython_api([rffi.CCHARP, rffi.CCHARPP, rffi.INT_real], PyObject) def PyInt_FromString(space, str, pend, base): """Return a new PyIntObject or PyLongObject based on the string value in str, which is interpreted according to the radix in base. If @@ -3025,22 +2587,14 @@ returned in this case.""" raise NotImplementedError - at cpython_api([Py_ssize_t], PyObject) -def PyInt_FromSsize_t(space, ival): - """Create a new integer object with a value of ival. If the value is larger - than LONG_MAX or smaller than LONG_MIN, a long integer object is - returned. - """ - raise NotImplementedError - - at cpython_api([{size_t}], PyObject) + at cpython_api([rffi.SIZE_T], PyObject) def PyInt_FromSize_t(space, ival): """Create a new integer object with a value of ival. If the value exceeds LONG_MAX, a long integer object is returned. """ raise NotImplementedError - at cpython_api([PyObject], {unsigned long}) + at cpython_api([PyObject], rffi.ULONG, error=-1) def PyInt_AsUnsignedLongMask(space, io): """Will first attempt to cast the object to a PyIntObject or PyLongObject, if it is not already one, and then return its value as @@ -3048,7 +2602,7 @@ """ raise NotImplementedError - at cpython_api([PyObject], {unsigned PY_LONG_LONG}) + at cpython_api([PyObject], rffi.LONGLONG, error=-1) def PyInt_AsUnsignedLongLongMask(space, io): """Will first attempt to cast the object to a PyIntObject or PyLongObject, if it is not already one, and then return its value as @@ -3056,47 +2610,27 @@ """ raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyInt_AsSsize_t(space, io): - """Will first attempt to cast the object to a PyIntObject or - PyLongObject, if it is not already one, and then return its value as - Py_ssize_t. - """ - raise NotImplementedError - - at cpython_api([], lltype.Signed) + at cpython_api([], lltype.Signed, error=CANNOT_FAIL) def PyInt_GetMax(space, ): """ - - - Return the system's idea of the largest integer it can handle (LONG_MAX, as defined in the system header files).""" raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyInt_ClearFreeList(space, ): """Clear the integer free list. Return the number of items that could not be freed. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyIter_Check(space, o): """Return true if the object o supports the iterator protocol.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyIter_Next(space, o): - """Return the next value from the iteration o. If the object is an iterator, - this retrieves the next value from the iteration, and returns NULL with no - exception set if there are no remaining items. If the object is not an - iterator, TypeError is raised, or if there is an error in retrieving the - item, returns NULL and passes along the exception.""" - raise NotImplementedError - - at cpython_api([{op}], rffi.INT_real) -def PySeqIter_Check(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PySeqIter_Check(space, seq): """Return true if the type of op is PySeqIter_Type. """ raise NotImplementedError @@ -3109,45 +2643,12 @@ """ raise NotImplementedError - at cpython_api([{op}], rffi.INT_real) -def PyCallIter_Check(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyCallIter_Check(space, iter): """Return true if the type of op is PyCallIter_Type. """ raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyCallIter_New(space, callable, sentinel): - """Return a new iterator. The first parameter, callable, can be any Python - callable object that can be called with no parameters; each call to it should - return the next item in the iteration. When callable returns a value equal to - sentinel, the iteration will be terminated. - """ - raise NotImplementedError - - at cpython_api([PyObject], Py_ssize_t) -def PyList_Size(space, list): - """ - - - - Return the length of the list object in list; this is equivalent to - len(list) on a list object. - - This function returned an int. This might require changes in - your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) -def PyList_GetItem(space, list, index): - """Return the object at position pos in the list pointed to by p. The - position must be positive, indexing from the end of the list is not - supported. If pos is out of bounds, return NULL and set an - IndexError exception. - - This function used an int for index. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) def PyList_GET_ITEM(space, list, i): """Macro form of PyList_GetItem() without error checking. @@ -3170,16 +2671,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real) -def PyList_Insert(space, list, index, item): - """Insert the item item into list list in front of index index. Return - 0 if successful; return -1 and set an exception if unsuccessful. - Analogous to list.insert(index, item). - - This function used an int for index. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PyList_GetSlice(space, list, low, high): """Return a list of the objects in list containing the objects between low @@ -3191,18 +2682,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real) -def PyList_SetSlice(space, list, low, high, itemlist): - """Set the slice of list between low and high to the contents of - itemlist. Analogous to list[low:high] = itemlist. The itemlist may - be NULL, indicating the assignment of an empty list (slice deletion). - Return 0 on success, -1 on failure. Negative indices, as when - slicing from Python, are not supported. - - This function used an int for low and high. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyList_AsTuple(space, list): """ @@ -3213,27 +2692,6 @@ tuple(list).""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyLong_Check(space, p): - """Return true if its argument is a PyLongObject or a subtype of - PyLongObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyLong_CheckExact(space, p): - """Return true if its argument is a PyLongObject, but not a subtype of - PyLongObject. - """ - raise NotImplementedError - - at cpython_api([{unsigned long}], PyObject) -def PyLong_FromUnsignedLong(space, v): - """Return a new PyLongObject object from a C unsigned long, or - NULL on failure.""" - raise NotImplementedError - @cpython_api([Py_ssize_t], PyObject) def PyLong_FromSsize_t(space, v): """Return a new PyLongObject object from a C Py_ssize_t, or @@ -3241,14 +2699,14 @@ """ raise NotImplementedError - at cpython_api([{size_t}], PyObject) + at cpython_api([rffi.SIZE_T], PyObject) def PyLong_FromSize_t(space, v): """Return a new PyLongObject object from a C size_t, or NULL on failure. """ raise NotImplementedError - at cpython_api([{Py_UNICODE*}, Py_ssize_t, rffi.INT_real], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject) def PyLong_FromUnicode(space, u, length, base): """Convert a sequence of Unicode digits to a Python long integer value. The first parameter, u, points to the first character of the Unicode string, length @@ -3262,17 +2720,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{void*}], PyObject) -def PyLong_FromVoidPtr(space, p): - """Create a Python integer or long integer from the pointer p. The pointer value - can be retrieved from the resulting value using PyLong_AsVoidPtr(). - - - - If the integer is larger than LONG_MAX, a positive long integer is returned.""" - raise NotImplementedError - - at cpython_api([PyObject, {int*}], lltype.Signed) + at cpython_api([PyObject, rffi.INTP], lltype.Signed, error=-1) def PyLong_AsLongAndOverflow(space, pylong, overflow): """Return a C long representation of the contents of pylong. If pylong is greater than LONG_MAX or less @@ -3284,7 +2732,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, {int*}], {PY_LONG_LONG}) + at cpython_api([PyObject, rffi.INTP], rffi.LONGLONG, error=-1) def PyLong_AsLongLongAndOverflow(space, pylong, overflow): """Return a C long long representation of the contents of pylong. If pylong is greater than PY_LLONG_MAX or less @@ -3308,54 +2756,14 @@ """ raise NotImplementedError - at cpython_api([PyObject], {unsigned long}) -def PyLong_AsUnsignedLong(space, pylong): - """ - - - - Return a C unsigned long representation of the contents of pylong. - If pylong is greater than ULONG_MAX, an OverflowError is - raised.""" - raise NotImplementedError - - at cpython_api([PyObject], {PY_LONG_LONG}) -def PyLong_AsLongLong(space, pylong): - """ - - - - Return a C long long from a Python long integer. If - pylong cannot be represented as a long long, an - OverflowError is raised and -1 is returned. - """ - raise NotImplementedError - - at cpython_api([PyObject], {unsigned PY_LONG_LONG}) -def PyLong_AsUnsignedLongLong(space, pylong): - """ - - - - Return a C unsigned long long from a Python long integer. If - pylong cannot be represented as an unsigned long long, an - OverflowError is raised and (unsigned long long)-1 is - returned. - - - - A negative pylong now raises OverflowError, not - TypeError.""" - raise NotImplementedError - - at cpython_api([PyObject], {unsigned long}) + at cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLongMask(space, io): """Return a C unsigned long from a Python long integer, without checking for overflow. """ raise NotImplementedError - at cpython_api([PyObject], {unsigned PY_LONG_LONG}) + at cpython_api([PyObject], rffi.ULONGLONG, error=-1) def PyLong_AsUnsignedLongLongMask(space, io): """Return a C unsigned long long from a Python long integer, without checking for overflow. @@ -3376,26 +2784,26 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyMapping_DelItemString(space, o, key): """Remove the mapping for object key from the object o. Return -1 on failure. This is equivalent to the Python statement del o[key].""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyMapping_DelItem(space, o, key): """Remove the mapping for object key from the object o. Return -1 on failure. This is equivalent to the Python statement del o[key].""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=CANNOT_FAIL) def PyMapping_HasKeyString(space, o, key): """On success, return 1 if the mapping object has the key key and 0 otherwise. This is equivalent to o[key], returning True on success and False on an exception. This function always succeeds.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyMapping_HasKey(space, o, key): """Return 1 if the mapping object has the key key and 0 otherwise. This is equivalent to o[key], returning True on success and False @@ -3408,19 +2816,7 @@ NULL. This is equivalent to the Python expression o.values().""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyMapping_GetItemString(space, o, key): - """Return element of o corresponding to the object key or NULL on failure. - This is the equivalent of the Python expression o[key].""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real) -def PyMapping_SetItemString(space, o, key, v): - """Map the object key to the value v in object o. Returns -1 on failure. - This is the equivalent of the Python statement o[key] = v.""" - raise NotImplementedError - - at cpython_api([lltype.Signed, {FILE*}, rffi.INT_real], lltype.Void) + at cpython_api([lltype.Signed, FILE, rffi.INT_real], lltype.Void) def PyMarshal_WriteLongToFile(space, value, file, version): """Marshal a long integer, value, to file. This will only write the least-significant 32 bits of value; regardless of the size of the @@ -3429,7 +2825,7 @@ version indicates the file format.""" raise NotImplementedError - at cpython_api([PyObject, {FILE*}, rffi.INT_real], lltype.Void) + at cpython_api([PyObject, FILE, rffi.INT_real], lltype.Void) def PyMarshal_WriteObjectToFile(space, value, file, version): """Marshal a Python object, value, to file. @@ -3443,28 +2839,28 @@ version indicates the file format.""" raise NotImplementedError - at cpython_api([{FILE*}], lltype.Signed) + at cpython_api([FILE], lltype.Signed, error=-1) def PyMarshal_ReadLongFromFile(space, file): """Return a C long from the data stream in a FILE* opened for reading. Only a 32-bit value can be read in using this function, regardless of the native size of long.""" raise NotImplementedError - at cpython_api([{FILE*}], rffi.INT_real) + at cpython_api([FILE], rffi.INT_real, error=-1) def PyMarshal_ReadShortFromFile(space, file): """Return a C short from the data stream in a FILE* opened for reading. Only a 16-bit value can be read in using this function, regardless of the native size of short.""" raise NotImplementedError - at cpython_api([{FILE*}], PyObject) + at cpython_api([FILE], PyObject) def PyMarshal_ReadObjectFromFile(space, file): """Return a Python object from the data stream in a FILE* opened for reading. On error, sets the appropriate exception (EOFError or TypeError) and returns NULL.""" raise NotImplementedError - at cpython_api([{FILE*}], PyObject) + at cpython_api([FILE], PyObject) def PyMarshal_ReadLastObjectFromFile(space, file): """Return a Python object from the data stream in a FILE* opened for reading. Unlike PyMarshal_ReadObjectFromFile(), this function @@ -3487,57 +2883,8 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{size_t}], {void*}) -def PyMem_Malloc(space, n): - """Allocates n bytes and returns a pointer of type void* to the - allocated memory, or NULL if the request fails. Requesting zero bytes returns - a distinct non-NULL pointer if possible, as if PyMem_Malloc(1)() had - been called instead. The memory will not have been initialized in any way.""" - raise NotImplementedError - - at cpython_api([{void*}, {size_t}], {void*}) -def PyMem_Realloc(space, p, n): - """Resizes the memory block pointed to by p to n bytes. The contents will be - unchanged to the minimum of the old and the new sizes. If p is NULL, the - call is equivalent to PyMem_Malloc(n)(); else if n is equal to zero, - the memory block is resized but is not freed, and the returned pointer is - non-NULL. Unless p is NULL, it must have been returned by a previous call - to PyMem_Malloc() or PyMem_Realloc(). If the request fails, - PyMem_Realloc() returns NULL and p remains a valid pointer to the - previous memory area.""" - raise NotImplementedError - - at cpython_api([{void*}], lltype.Void) -def PyMem_Free(space, p): - """Frees the memory block pointed to by p, which must have been returned by a - previous call to PyMem_Malloc() or PyMem_Realloc(). Otherwise, or - if PyMem_Free(p)() has been called before, undefined behavior occurs. If - p is NULL, no operation is performed.""" - raise NotImplementedError - - at cpython_api([{TYPE}, {size_t}], {TYPE*}) -def PyMem_New(space, , n): - """Same as PyMem_Malloc(), but allocates (n * sizeof(TYPE)) bytes of - memory. Returns a pointer cast to TYPE*. The memory will not have - been initialized in any way.""" - raise NotImplementedError - - at cpython_api([{void*}, {TYPE}, {size_t}], {TYPE*}) -def PyMem_Resize(space, p, , n): - """Same as PyMem_Realloc(), but the memory block is resized to (n * - sizeof(TYPE)) bytes. Returns a pointer cast to TYPE*. On return, - p will be a pointer to the new memory area, or NULL in the event of - failure. This is a C preprocessor macro; p is always reassigned. Save - the original value of p to avoid losing memory when handling errors.""" - raise NotImplementedError - - at cpython_api([{void*}], lltype.Void) -def PyMem_Del(space, p): - """Same as PyMem_Free().""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyMethod_New(space, func, self, class): +def PyMethod_New(space, func, self, cls): """Return a new method object, with func being any callable object; this is the function that will be called when the method is called. If this method should be bound to an instance, self should be the instance and class should be the @@ -3577,13 +2924,13 @@ """Macro version of PyMethod_Self() which avoids error checking.""" raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyMethod_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyModule_CheckExact(space, p): """Return true if p is a module object, but not a subtype of PyModule_Type. @@ -3601,18 +2948,15 @@ the caller is responsible for providing a __file__ attribute.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=lltype.nullptr(rffi.CCHARP.TO)) def PyModule_GetFilename(space, module): """ - - - Return the name of the file from which module was loaded using module's __file__ attribute. If this is not defined, or if it is not a string, raise SystemError and return NULL.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real, error=-1) def PyModule_AddObject(space, module, name, value): """Add an object to module as name. This is a convenience function which can be used from the module's initialization function. This steals a reference to @@ -3620,7 +2964,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, lltype.Signed], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, lltype.Signed], rffi.INT_real, error=-1) def PyModule_AddIntConstant(space, module, name, value): """Add an integer constant to module as name. This convenience function can be used from the module's initialization function. Return -1 on error, 0 on @@ -3628,7 +2972,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], rffi.INT_real, error=-1) def PyModule_AddStringConstant(space, module, name, value): """Add a string constant to module as name. This convenience function can be used from the module's initialization function. The string value must be @@ -3636,45 +2980,9 @@ """ raise NotImplementedError - at cpython_api([PyObject, {macro}], rffi.INT_real) -def PyModule_AddIntMacro(space, module, ): - """Add an int constant to module. The name and the value are taken from - macro. For example PyModule_AddConstant(module, AF_INET) adds the int - constant AF_INET with the value of AF_INET to module. - Return -1 on error, 0 on success. - """ - raise NotImplementedError - - at cpython_api([PyObject, {macro}], rffi.INT_real) -def PyModule_AddStringMacro(space, module, ): - """Add a string constant to module. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyNumber_Check(space, o): - """Returns 1 if the object o provides numeric protocols, and false otherwise. - This function always succeeds.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Add(space, o1, o2): - """Returns the result of adding o1 and o2, or NULL on failure. This is the - equivalent of the Python expression o1 + o2.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PyNumber_Subtract(space, o1, o2): - """Returns the result of subtracting o2 from o1, or NULL on failure. This is - the equivalent of the Python expression o1 - o2.""" - raise NotImplementedError - - at cpython_api([PyObjectP, PyObjectP], rffi.INT_real) + at cpython_api([PyObjectP, PyObjectP], rffi.INT_real, error=-1) def PyNumber_Coerce(space, p1, p2): """ - - - This function takes the addresses of two variables of type PyObject*. If the objects pointed to by *p1 and *p2 have the same type, increment their reference count and return 0 (success). If the objects can be @@ -3685,7 +2993,7 @@ &o2) is equivalent to the Python statement o1, o2 = coerce(o1, o2).""" raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP], rffi.INT_real) + at cpython_api([PyObjectP, PyObjectP], rffi.INT_real, error=-1) def PyNumber_CoerceEx(space, p1, p2): """This function is similar to PyNumber_Coerce(), except that it returns 1 when the conversion is not possible and when no error is raised. @@ -3709,26 +3017,7 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], Py_ssize_t) -def PyNumber_AsSsize_t(space, o, exc): - """Returns o converted to a Py_ssize_t value if o can be interpreted as an - integer. If o can be converted to a Python int or long but the attempt to - convert to a Py_ssize_t value would raise an OverflowError, then the - exc argument is the type of exception that will be raised (usually - IndexError or OverflowError). If exc is NULL, then the - exception is cleared and the value is clipped to PY_SSIZE_T_MIN for a negative - integer or PY_SSIZE_T_MAX for a positive integer. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyIndex_Check(space, o): - """Returns True if o is an index integer (has the nb_index slot of the - tp_as_number structure filled in). - """ - raise NotImplementedError - - at cpython_api([PyObject, {const char**}, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_t], rffi.INT_real, error=-1) def PyObject_AsCharBuffer(space, obj, buffer, buffer_len): """Returns a pointer to a read-only memory location usable as character-based input. The obj argument must support the single-segment character buffer @@ -3742,7 +3031,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, {const void**}, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, rffi.VOIDPP, Py_ssize_t], rffi.INT_real, error=-1) def PyObject_AsReadBuffer(space, obj, buffer, buffer_len): """Returns a pointer to a read-only memory location containing arbitrary data. The obj argument must support the single-segment readable buffer @@ -3756,14 +3045,14 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_CheckReadBuffer(space, o): """Returns 1 if o supports the single-segment readable buffer interface. Otherwise returns 0. """ raise NotImplementedError - at cpython_api([PyObject, {void**}, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, rffi.VOIDPP, Py_ssize_t], rffi.INT_real, error=-1) def PyObject_AsWriteBuffer(space, obj, buffer, buffer_len): """Returns a pointer to a writeable memory location. The obj argument must support the single-segment, character buffer interface. On success, @@ -3776,14 +3065,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, {FILE*}, rffi.INT_real], rffi.INT_real) -def PyObject_Print(space, o, fp, flags): - """Print an object o, on file fp. Returns -1 on error. The flags argument - is used to enable certain printing options. The only option currently supported - is Py_PRINT_RAW; if given, the str() of the object is written - instead of the repr().""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], PyObject) def PyObject_GetAttr(space, o, attr_name): """Retrieve an attribute named attr_name from object o. Returns the attribute @@ -3791,57 +3072,30 @@ expression o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_DelAttr(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python statement del o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real) + at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyObject_DelAttrString(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python statement del o.attr_name.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.INT_real], PyObject) -def PyObject_RichCompare(space, o1, o2, opid): - """Compare the values of o1 and o2 using the operation specified by opid, - which must be one of Py_LT, Py_LE, Py_EQ, - Py_NE, Py_GT, or Py_GE, corresponding to <, - <=, ==, !=, >, or >= respectively. This is the equivalent of - the Python expression o1 op o2, where op is the operator corresponding - to opid. Returns the value of the comparison on success, or NULL on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real) -def PyObject_RichCompareBool(space, o1, o2, opid): - """Compare the values of o1 and o2 using the operation specified by opid, - which must be one of Py_LT, Py_LE, Py_EQ, - Py_NE, Py_GT, or Py_GE, corresponding to <, - <=, ==, !=, >, or >= respectively. Returns -1 on error, - 0 if the result is false, 1 otherwise. This is the equivalent of the - Python expression o1 op o2, where op is the operator corresponding to - opid.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, {int*}], rffi.INT_real) + at cpython_api([PyObject, PyObject, rffi.INTP], rffi.INT_real, error=-1) def PyObject_Cmp(space, o1, o2, result): """ - - - Compare the values of o1 and o2 using a routine provided by o1, if one exists, otherwise with a routine provided by o2. The result of the comparison is returned in result. Returns -1 on failure. This is the equivalent of the Python statement result = cmp(o1, o2).""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=None) def PyObject_Compare(space, o1, o2): """ - - - Compare the values of o1 and o2 using a routine provided by o1, if one exists, otherwise with a routine provided by o2. Returns the result of the comparison on success. On error, the value returned is undefined; use @@ -3871,13 +3125,7 @@ function.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyCallable_Check(space, o): - """Determine if the object o is callable. Return 1 if the object is callable - and 0 otherwise. This function always succeeds.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, ...], PyObject) + at cpython_api([PyObject, rffi.CCHARP, ], PyObject) def PyObject_CallFunction(space, callable, format, ): """ @@ -3892,7 +3140,7 @@ PyObject_CallFunctionObjArgs() is a faster alternative.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP, ...], PyObject) + at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP, ], PyObject) def PyObject_CallMethod(space, o, method, format, ): """Call the method named method of object o with a variable number of C arguments. The C arguments are described by a Py_BuildValue() format @@ -3903,8 +3151,8 @@ PyObject_CallMethodObjArgs() is a faster alternative.""" raise NotImplementedError - at cpython_api([PyObject, ..., {NULL}], PyObject) -def PyObject_CallFunctionObjArgs(space, callable, , ): + at cpython_api([PyObject, ], PyObject) +def PyObject_CallFunctionObjArgs(space, callable, ): """Call a callable Python object callable, with a variable number of PyObject* arguments. The arguments are provided as a variable number of parameters followed by NULL. Returns the result of the call on success, or @@ -3912,8 +3160,8 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject, ..., {NULL}], PyObject) -def PyObject_CallMethodObjArgs(space, o, name, , ): + at cpython_api([PyObject, PyObject, ], PyObject) +def PyObject_CallMethodObjArgs(space, o, name, ): """Calls a method of the object o, where the name of the method is given as a Python string object in name. It is called with a variable number of PyObject* arguments. The arguments are provided as a variable number @@ -3922,7 +3170,7 @@ """ raise NotImplementedError - at cpython_api([PyObject], lltype.Signed) + at cpython_api([PyObject], lltype.Signed, error=-1) def PyObject_Hash(space, o): """ @@ -3932,7 +3180,7 @@ This is the equivalent of the Python expression hash(o).""" raise NotImplementedError - at cpython_api([PyObject], lltype.Signed) + at cpython_api([PyObject], lltype.Signed, error=-1) def PyObject_HashNotImplemented(space, o): """Set a TypeError indicating that type(o) is not hashable and return -1. This function receives special treatment when stored in a tp_hash slot, @@ -3956,20 +3204,13 @@ count is needed.""" raise NotImplementedError - at cpython_api([PyObject, PyTypeObjectPtr], rffi.INT_real) -def PyObject_TypeCheck(space, o, type): - """Return true if the object o is of type type or a subtype of type. Both - parameters must be non-NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_SetItem(space, o, key, v): """Map the object key to the value v. Returns -1 on failure. This is the equivalent of the Python statement o[key] = v.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_DelItem(space, o, key): """Delete the mapping for key from o. Returns -1 on failure. This is the equivalent of the Python statement del o[key].""" @@ -3984,64 +3225,6 @@ is active then NULL is returned but PyErr_Occurred() will return false.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_GetIter(space, o): - """This is equivalent to the Python expression iter(o). It returns a new - iterator for the object argument, or the object itself if the object is already - an iterator. Raises TypeError and returns NULL if the object cannot be - iterated.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_INCREF(space, o): - """Increment the reference count for object o. The object must not be NULL; if - you aren't sure that it isn't NULL, use Py_XINCREF().""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_XINCREF(space, o): - """Increment the reference count for object o. The object may be NULL, in - which case the macro has no effect.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_DECREF(space, o): - """Decrement the reference count for object o. The object must not be NULL; if - you aren't sure that it isn't NULL, use Py_XDECREF(). If the reference - count reaches zero, the object's type's deallocation function (which must not be - NULL) is invoked. - - The deallocation function can cause arbitrary Python code to be invoked (e.g. - when a class instance with a __del__() method is deallocated). While - exceptions in such code are not propagated, the executed code has free access to - all Python global variables. This means that any object that is reachable from - a global variable should be in a consistent state before Py_DECREF() is - invoked. For example, code to delete an object from a list should copy a - reference to the deleted object in a temporary variable, update the list data - structure, and then call Py_DECREF() for the temporary variable.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_XDECREF(space, o): - """Decrement the reference count for object o. The object may be NULL, in - which case the macro has no effect; otherwise the effect is the same as for - Py_DECREF(), and the same warning applies.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_CLEAR(space, o): - """Decrement the reference count for object o. The object may be NULL, in - which case the macro has no effect; otherwise the effect is the same as for - Py_DECREF(), except that the argument is also set to NULL. The warning - for Py_DECREF() does not apply with respect to the object passed because - the macro carefully uses a temporary variable and sets the argument to NULL - before decrementing its reference count. - - It is a good idea to use this macro whenever decrementing the value of a - variable that might be traversed during garbage collection. - """ - raise NotImplementedError - @cpython_api([], PyObject, borrowed=True) def PyEval_GetBuiltins(space, ): """Return a dictionary of the builtins in the current execution frame, @@ -4060,30 +3243,30 @@ or NULL if no frame is currently executing.""" raise NotImplementedError - at cpython_api([], {PyFrameObject*}, borrowed=True) + at cpython_api([], PyFrameObject, borrowed=True) def PyEval_GetFrame(space, ): """Return the current thread state's frame, which is NULL if no frame is currently executing.""" raise NotImplementedError - at cpython_api([{PyFrameObject*}], rffi.INT_real) + at cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyEval_GetRestricted(space, ): """If there is a current frame and it is executing in restricted mode, return true, otherwise false.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyEval_GetFuncName(space, func): """Return the name of func if it is a function, class or instance object, else the name of funcs type.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyEval_GetFuncDesc(space, func): """Return a description string, depending on the type of func. Return values include "()" for functions and methods, " constructor", @@ -4092,18 +3275,6 @@ func.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PySequence_Check(space, o): - """Return 1 if the object provides sequence protocol, and 0 otherwise. - This function always succeeds.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) -def PySequence_Concat(space, o1, o2): - """Return the concatenation of o1 and o2 on success, and NULL on failure. - This is the equivalent of the Python expression o1 + o2.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Repeat(space, o, count): """Return the result of repeating sequence object o count times, or NULL on @@ -4130,16 +3301,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject) -def PySequence_GetItem(space, o, i): - """Return the ith element of o, or NULL on failure. This is the equivalent of - the Python expression o[i]. - - This function used an int type for i. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PySequence_SetItem(space, o, i, v): """Assign object v to the ith element of o. Returns -1 on failure. This is the equivalent of the Python statement o[i] = v. This function does @@ -4149,7 +3311,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) def PySequence_DelItem(space, o, i): """Delete the ith element of object o. Returns -1 on failure. This is the equivalent of the Python statement del o[i]. @@ -4158,7 +3320,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1) def PySequence_SetSlice(space, o, i1, i2, v): """Assign the sequence object v to the slice in sequence object o from i1 to i2. This is the equivalent of the Python statement o[i1:i2] = v. @@ -4167,7 +3329,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) def PySequence_DelSlice(space, o, i1, i2): """Delete the slice in sequence object o from i1 to i2. Returns -1 on failure. This is the equivalent of the Python statement del o[i1:i2]. @@ -4176,7 +3338,7 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], Py_ssize_t) + at cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Count(space, o, value): """Return the number of occurrences of value in o, that is, return the number of keys for which o[key] == value. On failure, return -1. This is @@ -4186,14 +3348,14 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PySequence_Contains(space, o, value): """Determine if o contains value. If an item in o is equal to value, return 1, otherwise return 0. On error, return -1. This is equivalent to the Python expression value in o.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], Py_ssize_t) + at cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Index(space, o, value): """Return the first index i for which o[i] == value. On error, return -1. This is equivalent to the Python expression o.index(value). @@ -4208,7 +3370,7 @@ returned list is guaranteed to be new.""" raise NotImplementedError - at cpython_api([PyObject], PyObjectP) + at cpython_api([PyObject], PyObjectP, error=lltype.nullptr(PyObjectP.TO)) def PySequence_Fast_ITEMS(space, o): """Return the underlying array of PyObject pointers. Assumes that o was returned by PySequence_Fast() and o is not NULL. @@ -4232,32 +3394,32 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PySet_Check(space, p): """Return true if p is a set object or an instance of a subtype. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrozenSet_Check(space, p): """Return true if p is a frozenset object or an instance of a subtype. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyAnySet_Check(space, p): """Return true if p is a set object, a frozenset object, or an instance of a subtype.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyAnySet_CheckExact(space, p): """Return true if p is a set object or a frozenset object but not an instance of a subtype.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrozenSet_CheckExact(space, p): """Return true if p is a frozenset object but not an instance of a subtype.""" @@ -4303,7 +3465,7 @@ """Macro form of PySet_Size() without error checking.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PySet_Contains(space, anyset, key): """Return 1 if found, 0 if not found, and -1 if an error is encountered. Unlike the Python __contains__() method, this function does not automatically @@ -4312,7 +3474,7 @@ set, frozenset, or an instance of a subtype.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PySet_Add(space, set, key): """Add key to a set instance. Does not apply to frozenset instances. Return 0 on success or -1 on failure. Raise a TypeError if @@ -4325,7 +3487,7 @@ values of brand new frozensets before they are exposed to other code.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PySet_Discard(space, set, key): """Return 1 if found and removed, 0 if not found (no action taken), and -1 if an error is encountered. Does not raise KeyError for missing keys. Raise a @@ -4343,16 +3505,11 @@ set or its subtype.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=-1) def PySet_Clear(space, set): """Empty an existing set of all elements.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PySlice_Check(space, ob): - """Return true if ob is a slice object; ob must not be NULL.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], PyObject) def PySlice_New(space, start, stop, step): """Return a new slice object with the given values. The start, stop, and @@ -4362,7 +3519,7 @@ the new object could not be allocated.""" raise NotImplementedError - at cpython_api([{PySliceObject*}, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObject, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1) def PySlice_GetIndices(space, slice, length, start, stop, step): """Retrieve the start, stop and step indices from the slice object slice, assuming a sequence of length length. Treats indices greater than @@ -4382,25 +3539,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{PySliceObject*}, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t, Py_ssize_t], rffi.INT_real) -def PySlice_GetIndicesEx(space, slice, length, start, stop, step, slicelength): - """Usable replacement for PySlice_GetIndices(). Retrieve the start, - stop, and step indices from the slice object slice assuming a sequence of - length length, and store the length of the slice in slicelength. Out - of bounds indices are clipped in a manner consistent with the handling of - normal slices. - - Returns 0 on success and -1 on error with exception set. - - - - This function used an int type for length and an - int * type for start, stop, step, and slicelength. This - might require changes in your code for properly supporting 64-bit - systems.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, ...], PyObject) + at cpython_api([rffi.CCHARP, ], PyObject) def PyString_FromFormat(space, format, ): """Take a C printf()-style format string and a variable number of arguments, calculate the size of the resulting Python string and return a string @@ -4548,7 +3687,7 @@ Support for "%lld" and "%llu" added.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {va_list}], PyObject) + at cpython_api([rffi.CCHARP, va_list], PyObject) def PyString_FromFormatV(space, format, vargs): """Identical to PyString_FromFormat() except that it takes exactly two arguments.""" @@ -4562,33 +3701,12 @@ your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], rffi.CCHARP) + at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) def PyString_AS_STRING(space, string): """Macro form of PyString_AsString() but without error checking. Only string objects are supported; no Unicode objects should be passed.""" raise NotImplementedError - at cpython_api([PyObject, {char**}, Py_ssize_t], rffi.INT_real) -def PyString_AsStringAndSize(space, obj, buffer, length): - """Return a NUL-terminated representation of the contents of the object obj - through the output variables buffer and length. - - The function accepts both string and Unicode objects as input. For Unicode - objects it returns the default encoded version of the object. If length is - NULL, the resulting buffer may not contain NUL characters; if it does, the - function returns -1 and a TypeError is raised. - - The buffer refers to an internal string buffer of obj, not a copy. The data - must not be modified in any way, unless the string was just created using - PyString_FromStringAndSize(NULL, size). It must not be deallocated. If - string is a Unicode object, this function computes the default encoding of - string and operates on that. If string is not a string object at all, - PyString_AsStringAndSize() returns -1 and raises TypeError. - - This function used an int * type for length. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObjectP], lltype.Void) def PyString_InternInPlace(space, string): """Intern the argument *string in place. The argument must be the address of a @@ -4664,7 +3782,7 @@ This function is not available in 3.x and does not have a PyBytes alias.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=CANNOT_FAIL) def Py_FdIsInteractive(space, fp, filename): """Return true (nonzero) if the standard I/O file fp with name filename is deemed interactive. This is the case for files for which isatty(fileno(fp)) @@ -4681,7 +3799,7 @@ to be called.""" raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyOS_CheckStack(space, ): """Return true when the interpreter runs out of stack space. This is a reliable check, but is only available when USE_STACKCHECK is defined (currently @@ -4690,7 +3808,7 @@ own code.""" raise NotImplementedError - at cpython_api([rffi.INT_real], {PyOS_sighandler_t}) + at cpython_api([rffi.INT_real], PyOS_sighandler_t) def PyOS_getsig(space, i): """Return the current signal handler for signal i. This is a thin wrapper around either sigaction() or signal(). Do not call those functions @@ -4698,7 +3816,7 @@ (*)(int).""" raise NotImplementedError - at cpython_api([rffi.INT_real, {PyOS_sighandler_t}], {PyOS_sighandler_t}) + at cpython_api([rffi.INT_real, PyOS_sighandler_t], PyOS_sighandler_t) def PyOS_setsig(space, i, h): """Set the signal handler for signal i to be h; return the old signal handler. This is a thin wrapper around either sigaction() or signal(). Do @@ -4706,20 +3824,13 @@ alias for void (*)(int).""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {FILE*}], {FILE*}) -def PySys_GetFile(space, name, def): + at cpython_api([rffi.CCHARP, FILE], FILE) +def PySys_GetFile(space, name, default): """Return the FILE* associated with the object name in the sys module, or def if name is not in the module or is not associated with a FILE*.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, PyObject], rffi.INT_real) -def PySys_SetObject(space, name, v): - """Set name in the sys module to v unless v is NULL, in which - case name is deleted from the sys module. Returns 0 on success, -1 - on error.""" - raise NotImplementedError - @cpython_api([], lltype.Void) def PySys_ResetWarnOptions(space, ): """Reset sys.warnoptions to an empty list.""" @@ -4737,7 +3848,7 @@ (: on Unix, ; on Windows).""" raise NotImplementedError - at cpython_api([rffi.CCHARP, ...], lltype.Void) + at cpython_api([rffi.CCHARP, ], lltype.Void) def PySys_WriteStdout(space, format, ): """Write the output string described by format to sys.stdout. No exceptions are raised, even if truncation occurs (see below). @@ -4754,7 +3865,7 @@ is written to the real (C level) stdout.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, ...], lltype.Void) + at cpython_api([rffi.CCHARP, ], lltype.Void) def PySys_WriteStderr(space, format, ): """As above, but write to sys.stderr or stderr instead.""" raise NotImplementedError @@ -4783,8 +3894,8 @@ standard C library function exit(status).""" raise NotImplementedError - at cpython_api([{void (*func)}], rffi.INT_real) -def Py_AtExit(space, ()): + at cpython_api([rffi.VOIDP], rffi.INT_real, error=-1) +def Py_AtExit(space, func): """ @@ -4798,7 +3909,7 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([Py_ssize_t, ...], PyObject) + at cpython_api([Py_ssize_t, ], PyObject) def PyTuple_Pack(space, n, ): """Return a new tuple object of size n, or NULL on failure. The tuple values are initialized to the subsequent n C arguments pointing to Python objects. @@ -4838,7 +3949,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real) + at cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) def _PyTuple_Resize(space, p, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there @@ -4857,27 +3968,14 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyTuple_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyType_Check(space, o): - """Return true if the object o is a type object, including instances of types - derived from the standard type object. Return false in all other cases.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyType_CheckExact(space, o): - """Return true if the object o is a type object, but not a subtype of the - standard type object. Return false in all other cases. - """ - raise NotImplementedError - - at cpython_api([], {unsigned int}) -def PyType_ClearCache(space, ): + at cpython_api([], rffi.UINT, error=CANNOT_FAIL) +def PyType_ClearCache(space): """Clear the internal lookup cache. Return the current version tag. """ raise NotImplementedError @@ -4890,114 +3988,72 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL) def PyType_HasFeature(space, o, feature): """Return true if the type object o sets the feature feature. Type features are denoted by single bit flags.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyType_IS_GC(space, o): """Return true if the type object includes support for the cycle detector; this tests the type flag Py_TPFLAGS_HAVE_GC. """ raise NotImplementedError - at cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real) -def PyType_IsSubtype(space, a, b): - """Return true if a is a subtype of b. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject) -def PyType_GenericAlloc(space, type, nitems): - """ - - This function used an int type for nitems. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject) def PyType_GenericNew(space, type, args, kwds): raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real) -def PyUnicode_Check(space, o): - """Return true if the object o is a Unicode object or an instance of a Unicode - subtype. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real) -def PyUnicode_CheckExact(space, o): - """Return true if the object o is a Unicode object, but not an instance of a - subtype. - """ - raise NotImplementedError - - at cpython_api([], rffi.INT_real) + at cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyUnicode_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. """ raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISTITLE(space, ch): """Return 1 or 0 depending on whether ch is a titlecase character.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISDIGIT(space, ch): """Return 1 or 0 depending on whether ch is a digit character.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISNUMERIC(space, ch): """Return 1 or 0 depending on whether ch is a numeric character.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISALPHA(space, ch): """Return 1 or 0 depending on whether ch is an alphabetic character.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], {Py_UNICODE}) + at cpython_api([Py_UNICODE], Py_UNICODE, error=CANNOT_FAIL) def Py_UNICODE_TOTITLE(space, ch): """Return the character ch converted to title case.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_TODECIMAL(space, ch): """Return the character ch converted to a decimal positive integer. Return -1 if this is not possible. This macro does not raise exceptions.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], rffi.INT_real) + at cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_TODIGIT(space, ch): """Return the character ch converted to a single digit integer. Return -1 if this is not possible. This macro does not raise exceptions.""" raise NotImplementedError - at cpython_api([{Py_UNICODE}], {double}) + at cpython_api([Py_UNICODE], rffi.DOUBLE, error=CANNOT_FAIL) def Py_UNICODE_TONUMERIC(space, ch): """Return the character ch converted to a double. Return -1.0 if this is not possible. This macro does not raise exceptions.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t], PyObject) -def PyUnicode_FromUnicode(space, u, size): - """Create a Unicode Object from the Py_UNICODE buffer u of the given size. u - may be NULL which causes the contents to be undefined. It is the user's - responsibility to fill in the needed data. The buffer is copied into the new - object. If the buffer is not NULL, the return value might be a shared object. - Therefore, modification of the resulting Unicode object is only allowed when u - is NULL. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) def PyUnicode_FromEncodedObject(space, obj, encoding, errors): """Coerce an encoded object obj to an Unicode object and return a reference with @@ -5021,7 +4077,7 @@ throughout the interpreter whenever coercion to Unicode is needed.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject) def PyUnicode_Encode(space, s, size, encoding, errors): """Encode the Py_UNICODE buffer of the given size and return a Python string object. encoding and errors have the same meaning as the parameters @@ -5055,7 +4111,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_EncodeUTF8(space, s, size, errors): """Encode the Py_UNICODE buffer of the given size using UTF-8 and return a Python string object. Return NULL if an exception was raised by the codec. @@ -5071,7 +4127,7 @@ by the codec.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, {int*}], PyObject) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) def PyUnicode_DecodeUTF32(space, s, size, errors, byteorder): """Decode length bytes from a UTF-32 encoded buffer string and return the corresponding Unicode object. errors (if non-NULL) defines the error @@ -5100,7 +4156,7 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, {int*}, Py_ssize_t], PyObject) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF32Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF32(). If consumed is not NULL, PyUnicode_DecodeUTF32Stateful() will not treat @@ -5110,7 +4166,7 @@ """ raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP, rffi.INT_real], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.INT_real], PyObject) def PyUnicode_EncodeUTF32(space, s, size, errors, byteorder): """Return a Python bytes object holding the UTF-32 encoded value of the Unicode data in s. Output is written according to the following byte order: @@ -5137,7 +4193,7 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, {int*}], PyObject) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject) def PyUnicode_DecodeUTF16(space, s, size, errors, byteorder): """Decode length bytes from a UTF-16 encoded buffer string and return the corresponding Unicode object. errors (if non-NULL) defines the error @@ -5167,7 +4223,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, {int*}, Py_ssize_t], PyObject) + at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If consumed is not NULL, PyUnicode_DecodeUTF16Stateful() will not treat @@ -5182,7 +4238,7 @@ properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP, rffi.INT_real], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.INT_real], PyObject) def PyUnicode_EncodeUTF16(space, s, size, errors, byteorder): """Return a Python string object holding the UTF-16 encoded value of the Unicode data in s. Output is written according to the following byte order: @@ -5220,7 +4276,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t], PyObject) def PyUnicode_EncodeUnicodeEscape(space, s, size): """Encode the Py_UNICODE buffer of the given size using Unicode-Escape and return a Python string object. Return NULL if an exception was raised by the @@ -5246,7 +4302,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_EncodeRawUnicodeEscape(space, s, size, errors): """Encode the Py_UNICODE buffer of the given size using Raw-Unicode-Escape and return a Python string object. Return NULL if an exception was raised by @@ -5272,7 +4328,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_EncodeLatin1(space, s, size, errors): """Encode the Py_UNICODE buffer of the given size using Latin-1 and return a Python string object. Return NULL if an exception was raised by the codec. @@ -5297,7 +4353,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_EncodeASCII(space, s, size, errors): """Encode the Py_UNICODE buffer of the given size using ASCII and return a Python string object. Return NULL if an exception was raised by the codec. @@ -5328,7 +4384,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, PyObject, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, PyObject, rffi.CCHARP], PyObject) def PyUnicode_EncodeCharmap(space, s, size, mapping, errors): """Encode the Py_UNICODE buffer of the given size using the given mapping object and return a Python string object. Return NULL if an @@ -5345,7 +4401,7 @@ exception was raised by the codec.""" raise NotImplementedError - at cpython_api([{const Py_UNICODE*}, Py_ssize_t, PyObject, rffi.CCHARP], PyObject) + at cpython_api([rffi.CWCHARP, Py_ssize_t, PyObject, rffi.CCHARP], PyObject) def PyUnicode_TranslateCharmap(space, s, size, table, errors): """Translate a Py_UNICODE buffer of the given length by applying a character mapping table to it and return the resulting Unicode object. Return @@ -5362,7 +4418,7 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real, rffi.CCHARP, {int*}], PyObject) + at cpython_api([rffi.CCHARP, rffi.INT_real, rffi.CCHARP, rffi.INTP], PyObject) def PyUnicode_DecodeMBCSStateful(space, s, size, errors, consumed): """If consumed is NULL, behave like PyUnicode_DecodeMBCS(). If consumed is not NULL, PyUnicode_DecodeMBCSStateful() will not decode @@ -5423,7 +4479,7 @@ Unicode string.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], rffi.INT_real, error=-1) def PyUnicode_Tailmatch(space, str, substr, start, end, direction): """Return 1 if substr matches str*[*start:end] at the given tail end (direction == -1 means to do a prefix match, direction == 1 a suffix match), @@ -5434,7 +4490,7 @@ systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t) + at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2) def PyUnicode_Find(space, str, substr, start, end, direction): """Return the first position of substr in str*[*start:end] using the given direction (direction == 1 means to do a forward search, direction == -1 a @@ -5467,13 +4523,13 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-2) def PyUnicode_Compare(space, left, right): """Compare two strings and return -1, 0, 1 for less than, equal, and greater than, respectively.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real) + at cpython_api([PyObject, PyObject, rffi.INT_real], PyObject) def PyUnicode_RichCompare(space, left, right, op): """Rich compare two unicode strings and return one of the following: @@ -5497,7 +4553,7 @@ format % args. The args argument must be a tuple.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyUnicode_Contains(space, container, element): """Check whether element is contained in container and return true or false accordingly. @@ -5506,7 +4562,7 @@ there was an error.""" raise NotImplementedError - at cpython_api([rffi.INT_real, {char**}], rffi.INT_real) + at cpython_api([rffi.INT_real, rffi.CCHARPP], rffi.INT_real, error=2) def Py_Main(space, argc, argv): """The main program for the standard interpreter. This is made available for programs which embed Python. The argc and argv parameters should be @@ -5522,25 +4578,25 @@ Py_InspectFlag is not set.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=-1) def PyRun_AnyFile(space, fp, filename): """This is a simplified interface to PyRun_AnyFileExFlags() below, leaving closeit set to 0 and flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_AnyFileFlags(space, fp, filename, flags): """This is a simplified interface to PyRun_AnyFileExFlags() below, leaving the closeit argument set to 0.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) def PyRun_AnyFileEx(space, fp, filename, closeit): """This is a simplified interface to PyRun_AnyFileExFlags() below, leaving the flags argument set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_AnyFileExFlags(space, fp, filename, closeit, flags): """If fp refers to a file associated with an interactive device (console or terminal input or Unix pseudo-terminal), return the value of @@ -5549,13 +4605,13 @@ "???" as the filename.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], rffi.INT_real) + at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1) def PyRun_SimpleString(space, command): """This is a simplified interface to PyRun_SimpleStringFlags() below, leaving the PyCompilerFlags* argument set to NULL.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([rffi.CCHARP, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_SimpleStringFlags(space, command, flags): """Executes the Python source code from command in the __main__ module according to the flags argument. If __main__ does not already exist, it @@ -5568,25 +4624,25 @@ Py_InspectFlag is not set.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=-1) def PyRun_SimpleFile(space, fp, filename): """This is a simplified interface to PyRun_SimpleFileExFlags() below, leaving closeit set to 0 and flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_SimpleFileFlags(space, fp, filename, flags): """This is a simplified interface to PyRun_SimpleFileExFlags() below, leaving closeit set to 0.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) def PyRun_SimpleFileEx(space, fp, filename, closeit): """This is a simplified interface to PyRun_SimpleFileExFlags() below, leaving flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_SimpleFileExFlags(space, fp, filename, closeit, flags): """Similar to PyRun_SimpleStringFlags(), but the Python source code is read from fp instead of an in-memory string. filename should be the name of the @@ -5594,13 +4650,13 @@ returns.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=-1) def PyRun_InteractiveOne(space, fp, filename): """This is a simplified interface to PyRun_InteractiveOneFlags() below, leaving flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_InteractiveOneFlags(space, fp, filename, flags): """Read and execute a single statement from a file associated with an interactive device according to the flags argument. If filename is NULL, "???" is @@ -5611,34 +4667,34 @@ not included by Python.h, so must be included specifically if needed.)""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=-1) def PyRun_InteractiveLoop(space, fp, filename): """This is a simplified interface to PyRun_InteractiveLoopFlags() below, leaving flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, {PyCompilerFlags*}], rffi.INT_real) + at cpython_api([FILE, rffi.CCHARP, PyCompilerFlags], rffi.INT_real, error=-1) def PyRun_InteractiveLoopFlags(space, fp, filename, flags): """Read and execute statements from a file associated with an interactive device until EOF is reached. If filename is NULL, "???" is used instead. The user will be prompted using sys.ps1 and sys.ps2. Returns 0 at EOF.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real], {struct _node*}) + at cpython_api([rffi.CCHARP, rffi.INT_real], _node) def PyParser_SimpleParseString(space, str, start): """This is a simplified interface to PyParser_SimpleParseStringFlagsFilename() below, leaving filename set to NULL and flags set to 0.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real, rffi.INT_real], {struct _node*}) + at cpython_api([rffi.CCHARP, rffi.INT_real, rffi.INT_real], _node) def PyParser_SimpleParseStringFlags(space, str, start, flags): """This is a simplified interface to PyParser_SimpleParseStringFlagsFilename() below, leaving filename set to NULL.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.INT_real], {struct _node*}) + at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.INT_real], _node) def PyParser_SimpleParseStringFlagsFilename(space, str, filename, start, flags): """Parse Python source code from str using the start token start according to the flags argument. The result can be used to create a code object which can @@ -5646,25 +4702,19 @@ many times.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real], {struct _node*}) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real], _node) def PyParser_SimpleParseFile(space, fp, filename, start): """This is a simplified interface to PyParser_SimpleParseFileFlags() below, leaving flags set to 0""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, rffi.INT_real], {struct _node*}) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, rffi.INT_real], _node) def PyParser_SimpleParseFileFlags(space, fp, filename, start, flags): """Similar to PyParser_SimpleParseStringFlagsFilename(), but the Python source code is read from fp instead of an in-memory string.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real, PyObject, PyObject], PyObject) -def PyRun_String(space, str, start, globals, locals): - """This is a simplified interface to PyRun_StringFlags() below, leaving - flags set to NULL.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, rffi.INT_real, PyObject, PyObject, {PyCompilerFlags*}], PyObject) + at cpython_api([rffi.CCHARP, rffi.INT_real, PyObject, PyObject, PyCompilerFlags], PyObject) def PyRun_StringFlags(space, str, start, globals, locals, flags): """Execute Python source code from str in the context specified by the dictionaries globals and locals with the compiler flags specified by @@ -5675,25 +4725,25 @@ exception was raised.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, PyObject, PyObject], PyObject) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject], PyObject) def PyRun_File(space, fp, filename, start, globals, locals): """This is a simplified interface to PyRun_FileExFlags() below, leaving closeit set to 0 and flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real], PyObject) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real], PyObject) def PyRun_FileEx(space, fp, filename, start, globals, locals, closeit): """This is a simplified interface to PyRun_FileExFlags() below, leaving flags set to NULL.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, {PyCompilerFlags*}], PyObject) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, PyCompilerFlags], PyObject) def PyRun_FileFlags(space, fp, filename, start, globals, locals, flags): """This is a simplified interface to PyRun_FileExFlags() below, leaving closeit set to 0.""" raise NotImplementedError - at cpython_api([{FILE*}, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real, {PyCompilerFlags*}], PyObject) + at cpython_api([FILE, rffi.CCHARP, rffi.INT_real, PyObject, PyObject, rffi.INT_real, PyCompilerFlags], PyObject) def PyRun_FileExFlags(space, fp, filename, start, globals, locals, closeit, flags): """Similar to PyRun_StringFlags(), but the Python source code is read from fp instead of an in-memory string. filename should be the name of the file. @@ -5707,7 +4757,7 @@ flags set to NULL.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real, {PyCompilerFlags*}], PyObject) + at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real, PyCompilerFlags], PyObject) def Py_CompileStringFlags(space, str, filename, start, flags): """Parse and compile the Python source code in str, returning the resulting code object. The start token is given by start; this can be used to constrain the @@ -5718,14 +4768,14 @@ be parsed or compiled.""" raise NotImplementedError - at cpython_api([{PyCodeObject*}, PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, co, globals, locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just the code object, and the dictionaries of global and local variables. The other arguments are set to NULL.""" raise NotImplementedError - at cpython_api([{PyCodeObject*}, PyObject, PyObject, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObject], PyObject) + at cpython_api([PyCodeObject, PyObject, PyObject, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObjectP, rffi.INT_real, PyObject], PyObject) def PyEval_EvalCodeEx(space, co, globals, locals, args, argcount, kws, kwcount, defs, defcount, closure): """Evaluate a precompiled code object, given a particular environment for its evaluation. This environment consists of dictionaries of global and local @@ -5733,13 +4783,13 @@ cells.""" raise NotImplementedError - at cpython_api([{PyFrameObject*}], PyObject) + at cpython_api([PyFrameObject], PyObject) def PyEval_EvalFrame(space, f): """Evaluate an execution frame. This is a simplified interface to PyEval_EvalFrameEx, for backward compatibility.""" raise NotImplementedError - at cpython_api([{PyFrameObject*}, rffi.INT_real], PyObject) + at cpython_api([PyFrameObject, rffi.INT_real], PyObject) def PyEval_EvalFrameEx(space, f, throwflag): """This is the main, unvarnished function of Python interpretation. It is literally 2000 lines long. The code object associated with the execution @@ -5749,26 +4799,26 @@ throw() methods of generator objects.""" raise NotImplementedError - at cpython_api([{PyCompilerFlags*}], rffi.INT_real) + at cpython_api([PyCompilerFlags], rffi.INT_real, error=0) def PyEval_MergeCompilerFlags(space, cf): """This function changes the flags of the current evaluation frame, and returns true on success, false on failure.""" raise NotImplementedError - at cpython_api([{ob}], rffi.INT_real) -def PyWeakref_Check(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyWeakref_Check(space, ref): """Return true if ob is either a reference or proxy object. """ raise NotImplementedError - at cpython_api([{ob}], rffi.INT_real) -def PyWeakref_CheckRef(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyWeakref_CheckRef(space, ref): """Return true if ob is a reference object. """ raise NotImplementedError - at cpython_api([{ob}], rffi.INT_real) -def PyWeakref_CheckProxy(space, ): + at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyWeakref_CheckProxy(space, ref): """Return true if ob is a proxy object. """ raise NotImplementedError From afa at codespeak.net Mon Apr 26 19:04:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 19:04:59 +0200 (CEST) Subject: [pypy-svn] r74088 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426170459.38C47282B9C@codespeak.net> Author: afa Date: Mon Apr 26 19:04:57 2010 New Revision: 74088 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: revert r74070, all symbols need to be renamed to avoid clashes with the hosting CPython. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 19:04:57 2010 @@ -489,7 +489,7 @@ def build_bridge(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = SYMBOLS_C + list(GLOBALS) + export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -692,7 +692,7 @@ def setup_library(space): from pypy.module.cpyext.pyobject import make_ref - export_symbols = SYMBOLS_C + list(GLOBALS) + export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) from pypy.translator.c.database import LowLevelDatabase db = LowLevelDatabase() From afa at codespeak.net Mon Apr 26 19:11:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 19:11:43 +0200 (CEST) Subject: [pypy-svn] r74089 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426171143.791CE282B9C@codespeak.net> Author: afa Date: Mon Apr 26 19:11:41 2010 New Revision: 74089 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Log: PyMapping_Size, PyMapping_Values, PyMapping_HasKey, PyMapping_HasKeyString Modified: pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/mapping.py Mon Apr 26 19:11:41 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, rffi -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING +from pypy.module.cpyext.api import ( + cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) from pypy.module.cpyext.pyobject import PyObject @@ -9,6 +10,14 @@ function always succeeds.""" return int(space.findattr(w_obj, space.wrap("items")) is not None) + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyMapping_Size(space, w_obj): + return space.int_w(space.len(w_obj)) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def PyMapping_Length(space, w_obj): + return space.int_w(space.len(w_obj)) + @cpython_api([PyObject], PyObject) def PyMapping_Keys(space, w_obj): """On success, return a list of the keys in object o. On failure, return NULL. @@ -16,6 +25,12 @@ return space.call_method(w_obj, "keys") @cpython_api([PyObject], PyObject) +def PyMapping_Values(space, w_obj): + """On success, return a list of the values in object o. On failure, return + NULL. This is equivalent to the Python expression o.values().""" + return space.call_method(w_obj, "values") + + at cpython_api([PyObject], PyObject) def PyMapping_Items(space, w_obj): """On success, return a list of the items in object o, where each item is a tuple containing a key-value pair. On failure, return NULL. This is equivalent to @@ -37,3 +52,26 @@ space.setitem(w_obj, w_key, w_value) return 0 + at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyMapping_HasKey(space, w_obj, w_key): + """Return 1 if the mapping object has the key key and 0 otherwise. + This is equivalent to o[key], returning True on success and False + on an exception. This function always succeeds.""" + try: + space.getitem(w_obj, w_key) + return 1 + except: + return 0 + + at cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=CANNOT_FAIL) +def PyMapping_HasKeyString(space, w_obj, key): + """Return 1 if the mapping object has the key key and 0 otherwise. + This is equivalent to o[key], returning True on success and False + on an exception. This function always succeeds.""" + try: + w_key = space.wrap(rffi.charp2str(key)) + space.getitem(w_obj, w_key) + return 1 + except: + return 0 + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 19:11:41 2010 @@ -2770,20 +2770,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyMapping_Size(space, o): - """ - - - - Returns the number of keys in object o on success, and -1 on failure. For - objects that do not provide mapping protocol, this is equivalent to the Python - expression len(o). - - These functions returned an int type. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyMapping_DelItemString(space, o, key): """Remove the mapping for object key from the object o. Return -1 on @@ -2796,26 +2782,6 @@ failure. This is equivalent to the Python statement del o[key].""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=CANNOT_FAIL) -def PyMapping_HasKeyString(space, o, key): - """On success, return 1 if the mapping object has the key key and 0 - otherwise. This is equivalent to o[key], returning True on success - and False on an exception. This function always succeeds.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyMapping_HasKey(space, o, key): - """Return 1 if the mapping object has the key key and 0 otherwise. - This is equivalent to o[key], returning True on success and False - on an exception. This function always succeeds.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyMapping_Values(space, o): - """On success, return a list of the values in object o. On failure, return - NULL. This is equivalent to the Python expression o.values().""" - raise NotImplementedError - @cpython_api([lltype.Signed, FILE, rffi.INT_real], lltype.Void) def PyMarshal_WriteLongToFile(space, value, file, version): """Marshal a long integer, value, to file. This will only write Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_mapping.py Mon Apr 26 19:11:41 2010 @@ -6,14 +6,19 @@ assert api.PyMapping_Check(space.newdict()) assert not api.PyMapping_Check(space.newlist([])) - def test_keys(self, space, api): + def test_size(self, space, api): w_d = space.newdict() - space.setitem(w_d, space.wrap("a"), space.wrap("a")) - assert space.eq_w(api.PyMapping_Keys(w_d), space.wrap(["a"])) + space.setitem(w_d, space.wrap("a"), space.wrap("b")) - def test_items(self, space, api): + assert api.PyMapping_Size(w_d) == 1 + assert api.PyMapping_Length(w_d) == 1 + + def test_keys(self, space, api): w_d = space.newdict() space.setitem(w_d, space.wrap("a"), space.wrap("b")) + + assert space.eq_w(api.PyMapping_Keys(w_d), space.wrap(["a"])) + assert space.eq_w(api.PyMapping_Values(w_d), space.wrap(["b"])) assert space.eq_w(api.PyMapping_Items(w_d), space.wrap([("a", "b")])) def test_setitemstring(self, space, api): @@ -23,3 +28,13 @@ assert 42 == space.unwrap( api.PyMapping_GetItemString(w_d, key, space.wrap(42))) rffi.free_charp(key) + + def test_haskey(self, space, api): + w_d = space.newdict() + space.setitem(w_d, space.wrap("a"), space.wrap("b")) + + assert api.PyMapping_HasKey(w_d, space.wrap("a")) + assert not api.PyMapping_HasKey(w_d, space.wrap("b")) + + assert api.PyMapping_HasKey(w_d, w_d) == 0 + # and no error is set From afa at codespeak.net Mon Apr 26 19:25:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 19:25:13 +0200 (CEST) Subject: [pypy-svn] r74090 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100426172513.0297C282B9C@codespeak.net> Author: afa Date: Mon Apr 26 19:25:12 2010 New Revision: 74090 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Log: PySeqIter_New Modified: pypy/branch/cpython-extension/pypy/module/cpyext/number.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/number.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/number.py Mon Apr 26 19:25:12 2010 @@ -46,7 +46,7 @@ @cpython_api([PyObject], PyObject) def PyNumber_Long(space, w_obj): - """ Returns the o converted to a long integer object on success, or NULL on + """Returns the o converted to a long integer object on success, or NULL on failure. This is the equivalent of the Python expression long(o).""" return space.long(w_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/sequence.py Mon Apr 26 19:25:12 2010 @@ -93,3 +93,12 @@ """Return the concatenation of o1 and o2 on success, and NULL on failure. This is the equivalent of the Python expression o1 + o2.""" return space.add(w_o1, w_o2) + + at cpython_api([PyObject], PyObject) +def PySeqIter_New(space, w_seq): + """Return an iterator that works with a general sequence object, seq. The + iteration ends when the sequence raises IndexError for the subscripting + operation. + """ + return space.iter(w_seq) + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Mon Apr 26 19:25:12 2010 @@ -2635,14 +2635,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PySeqIter_New(space, seq): - """Return an iterator that works with a general sequence object, seq. The - iteration ends when the sequence raises IndexError for the subscripting - operation. - """ - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyCallIter_Check(space, iter): """Return true if the type of op is PyCallIter_Type. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_sequence.py Mon Apr 26 19:25:12 2010 @@ -2,6 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext import sequence +import py.test class TestSequence(BaseApiTest): def test_sequence(self, space, api): @@ -43,3 +44,11 @@ w_t = space.wrap((1, 2, 3, 4, 5)) assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == (3, 4) assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == (2, 3, 4) + + def test_iter(self, space, api): + w_t = space.wrap((1, 2)) + w_iter = api.PySeqIter_New(w_t) + assert space.unwrap(space.next(w_iter)) == 1 + assert space.unwrap(space.next(w_iter)) == 2 + exc = raises(OperationError, space.next, w_iter) + assert exc.value.match(space, space.w_StopIteration) From afa at codespeak.net Mon Apr 26 19:49:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 26 Apr 2010 19:49:06 +0200 (CEST) Subject: [pypy-svn] r74091 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426174906.1F385282B9C@codespeak.net> Author: afa Date: Mon Apr 26 19:49:04 2010 New Revision: 74091 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Provide an actual definition for structures like PyIntObject, boost::python needs it to subclass types. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Mon Apr 26 19:49:04 2010 @@ -297,7 +297,8 @@ GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) for cpyname in 'Method List Int Long Dict Tuple'.split(): - FORWARD_DECLS.append('struct Py%sObject' % (cpyname, )) + FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' + 'Py%sObject' % (cpyname, )) build_exported_objects() def get_structtype_for_ctype(ctype): From fijal at codespeak.net Mon Apr 26 20:08:34 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Apr 2010 20:08:34 +0200 (CEST) Subject: [pypy-svn] r74092 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100426180834.93A67282B9C@codespeak.net> Author: fijal Date: Mon Apr 26 20:08:31 2010 New Revision: 74092 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Write down a test for exc exit Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Mon Apr 26 20:08:31 2010 @@ -202,3 +202,24 @@ L6: int_return $54 """) + + def test_exc_exitswitch(self): + py.test.skip("not implemented") + def g(i): + if i == 2: + raise ValueError + elif i == 3: + raise KeyError + + def f(i): + try: + g(i) + except ValueError: + return 1 + except KeyError: + return 2 + else: + return 3 + + self.encoding_test(f, [65], """ + """) From afa at codespeak.net Tue Apr 27 00:32:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 27 Apr 2010 00:32:09 +0200 (CEST) Subject: [pypy-svn] r74093 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100426223209.90CC3282B9D@codespeak.net> Author: afa Date: Tue Apr 27 00:32:05 2010 New Revision: 74093 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: update TODO list Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Tue Apr 27 00:32:05 2010 @@ -13,9 +13,8 @@ - replace @cpython_api(external=False) by another explicit name: all it does is a lltype function pointer, no C code involved. - - Fix distutil's build_ext to work with cpyext on windows. - - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). + - Fix PyEval_SaveThread: http://paste.pocoo.org/show/206521/ - Make Numpy work. From fijal at codespeak.net Tue Apr 27 04:47:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 04:47:40 +0200 (CEST) Subject: [pypy-svn] r74094 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427024740.2194F282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 04:47:39 2010 New Revision: 74094 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Write down a first test for exceptions. Incredible algorithmic knowledge needed for string comparison Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 04:47:39 2010 @@ -1,4 +1,4 @@ -from pypy.objspace.flow.model import Variable, Constant +from pypy.objspace.flow.model import Variable, Constant, c_last_exception from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.rpython.lltypesystem import lltype @@ -158,6 +158,29 @@ self.emitline(Label(linkfalse)) self.make_link(linkfalse) # + elif block.exitswitch is c_last_exception: + # An exception block. Would create something like: + # if exception jump first check + # defaultcase + # if not exc_1 jmp next check + # exc_1 case + # if not exc_2 jmp next check + # exc_2 case + # reraise + assert block.exits[0].exitcase is None # is this always True? + self.emitline('goto_if_exception', TLabel(block.exits[0])) + self.make_link(block.exits[0]) + self.emitline(Label(block.exits[0])) + for link in block.exits[1:]: + if (link.exitcase is Exception and link.target.operations == () + and len(link.target.inputargs) == 2): + # default exit-by-exception block + self.emitline("reraise") + else: + self.emitline('goto_if_exception_mismatch', + Constant(link.llexitcase), TLabel(link)) + self.make_link(link) + self.emitline(Label(link)) else: # A switch. # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 04:47:39 2010 @@ -53,9 +53,45 @@ self.assert_format(ssarepr, expected) def assert_format(self, ssarepr, expected): + def replace_struct(s): + """ Replace all $<* struct ...> with $STRUCT + """ + while True: + i = s.find('$<* struct') + if i == -1: + return s + count = 1 + start = i + i += 2 + while True: + if s[i] == '<': + count += 1 + if s[i] == '>': + count -= 1 + if count == 0: + break + i += 1 + s = s[:start] + '$STRUCT' + s[i + 1:] + asm = format_assembler(ssarepr) expected = str(py.code.Source(expected)).strip() + '\n' - assert asm == expected + asmlines = asm.split("\n") + explines = expected.split("\n") + for asm, exp in zip(asmlines, explines): + asm = replace_struct(asm) + if asm != exp: + print + print "Got: " + asm + print "Expected: " + exp + lgt = 0 + for i in range(len(asm)): + if exp[i] == asm[i]: + lgt += 1 + else: + break + print " " + " " * lgt + "^^^^" + raise AssertionError + assert len(asmlines) == len(explines) def test_simple(self): def f(n): @@ -204,12 +240,8 @@ """) def test_exc_exitswitch(self): - py.test.skip("not implemented") def g(i): - if i == 2: - raise ValueError - elif i == 3: - raise KeyError + pass def f(i): try: @@ -222,4 +254,15 @@ return 3 self.encoding_test(f, [65], """ + direct_call $<* fn g>, %i0 + goto_if_exception L1 + int_return $3 + L1: + goto_if_exception_mismatch $STRUCT, L2 + int_return $1 + L2: + goto_if_exception_mismatch $STRUCT, L3 + int_return $2 + L3: + reraise """) From fijal at codespeak.net Tue Apr 27 04:58:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 04:58:46 +0200 (CEST) Subject: [pypy-svn] r74095 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100427025846.9F648282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 04:58:45 2010 New Revision: 74095 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Log: Leave a comment. Also don't rely on whether operations is an empty tuple or empty list Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 04:58:45 2010 @@ -172,13 +172,17 @@ self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) for link in block.exits[1:]: - if (link.exitcase is Exception and link.target.operations == () + if (link.exitcase is Exception and (not link.target.operations) and len(link.target.inputargs) == 2): # default exit-by-exception block self.emitline("reraise") else: self.emitline('goto_if_exception_mismatch', - Constant(link.llexitcase), TLabel(link)) + Constant(link.llexitcase), + # XXX should I live concretetype here + # or should I just make user do + # lltype.typeOf??? + TLabel(link)) self.make_link(link) self.emitline(Label(link)) else: From benjamin at codespeak.net Tue Apr 27 05:16:40 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 27 Apr 2010 05:16:40 +0200 (CEST) Subject: [pypy-svn] r74096 - pypy/trunk/pypy/rpython Message-ID: <20100427031640.4D14A282B9D@codespeak.net> Author: benjamin Date: Tue Apr 27 05:16:38 2010 New Revision: 74096 Modified: pypy/trunk/pypy/rpython/rtyper.py Log: remove unused list_of_str_repr attribute Modified: pypy/trunk/pypy/rpython/rtyper.py ============================================================================== --- pypy/trunk/pypy/rpython/rtyper.py (original) +++ pypy/trunk/pypy/rpython/rtyper.py Tue Apr 27 05:16:38 2010 @@ -199,7 +199,7 @@ assert dont_simplify_again in (False, True) # safety check if not dont_simplify_again: self.annotator.simplify() - + # first make sure that all functions called in a group have exactly # the same signature, by hacking their flow graphs if needed self.type_system.perform_normalizations(self) @@ -214,11 +214,6 @@ self.specialize_more_blocks() # for the helpers just made if self.type_system.name == 'ootypesystem': self.attach_methods_to_subclasses() - - # - from pypy.annotation import listdef - ldef = listdef.ListDef(None, annmodel.SomeString()) - self.list_of_str_repr = self.getrepr(annmodel.SomeList(ldef)) def getannmixlevel(self): if self.annmixlevel is not None: From fijal at codespeak.net Tue Apr 27 06:09:17 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 06:09:17 +0200 (CEST) Subject: [pypy-svn] r74097 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100427040917.23054282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 06:09:15 2010 New Revision: 74097 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Log: I'm not sure if it's correct, but use lltype.typeOf here Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 06:09:15 2010 @@ -178,10 +178,8 @@ self.emitline("reraise") else: self.emitline('goto_if_exception_mismatch', - Constant(link.llexitcase), - # XXX should I live concretetype here - # or should I just make user do - # lltype.typeOf??? + Constant(link.llexitcase, + lltype.typeOf(link.llexitcase))), TLabel(link)) self.make_link(link) self.emitline(Label(link)) From fijal at codespeak.net Tue Apr 27 06:11:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 06:11:35 +0200 (CEST) Subject: [pypy-svn] r74098 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100427041135.E55C3282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 06:11:34 2010 New Revision: 74098 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Log: ekhem, stop checking in syntaxerrors Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 06:11:34 2010 @@ -179,7 +179,7 @@ else: self.emitline('goto_if_exception_mismatch', Constant(link.llexitcase, - lltype.typeOf(link.llexitcase))), + lltype.typeOf(link.llexitcase)), TLabel(link)) self.make_link(link) self.emitline(Label(link)) From afa at codespeak.net Tue Apr 27 10:32:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 27 Apr 2010 10:32:36 +0200 (CEST) Subject: [pypy-svn] r74100 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100427083236.3098D282BDB@codespeak.net> Author: afa Date: Tue Apr 27 10:32:34 2010 New Revision: 74100 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_weakref.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/weakrefobject.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: PyWeakref_NewRef, PyWeakref_GetObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Tue Apr 27 10:32:34 2010 @@ -66,6 +66,7 @@ import pypy.module.cpyext.pystate import pypy.module.cpyext.datetime import pypy.module.cpyext.complexobject +import pypy.module.cpyext.weakrefobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Tue Apr 27 10:32:34 2010 @@ -4782,19 +4782,6 @@ raise NotImplementedError @cpython_api([PyObject, PyObject], PyObject) -def PyWeakref_NewRef(space, ob, callback): - """Return a weak reference object for the object ob. This will always return - a new reference, but is not guaranteed to create a new object; an existing - reference object may be returned. The second parameter, callback, can be a - callable object that receives notification when ob is garbage collected; it - should accept a single parameter, which will be the weak reference object - itself. callback may also be None or NULL. If ob is not a - weakly-referencable object, or if callback is not callable, None, or - NULL, this will return NULL and raise TypeError. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject], PyObject) def PyWeakref_NewProxy(space, ob, callback): """Return a weak reference proxy object for the object ob. This will always return a new reference, but is not guaranteed to create a new object; an @@ -4808,13 +4795,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject, borrowed=True) -def PyWeakref_GetObject(space, ref): - """Return the referenced object from a weak reference, ref. If the referent is - no longer live, returns None. - """ - raise NotImplementedError - - at cpython_api([PyObject], PyObject, borrowed=True) def PyWeakref_GET_OBJECT(space, ref): """Similar to PyWeakref_GetObject(), but implemented as a macro that does no error checking. Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_weakref.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_weakref.py Tue Apr 27 10:32:34 2010 @@ -0,0 +1,14 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest + +class TestWeakReference(BaseApiTest): + def test_weakref(self, space, api): + w_obj = space.w_Exception + w_ref = api.PyWeakref_NewRef(w_obj, space.w_None) + assert w_ref is not None + assert space.is_w(api.PyWeakref_GetObject(w_ref), w_obj) + + w_obj = space.newtuple([]) + assert api.PyWeakref_NewRef(w_obj, space.w_None) is None + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() Added: pypy/branch/cpython-extension/pypy/module/cpyext/weakrefobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/weakrefobject.py Tue Apr 27 10:32:34 2010 @@ -0,0 +1,26 @@ +from pypy.module.cpyext.api import cpython_api +from pypy.module.cpyext.pyobject import PyObject, register_container +from pypy.module._weakref.interp__weakref import W_Weakref + + at cpython_api([PyObject, PyObject], PyObject) +def PyWeakref_NewRef(space, w_obj, w_callback): + """Return a weak reference object for the object ob. This will always return + a new reference, but is not guaranteed to create a new object; an existing + reference object may be returned. The second parameter, callback, can be a + callable object that receives notification when ob is garbage collected; it + should accept a single parameter, which will be the weak reference object + itself. callback may also be None or NULL. If ob is not a + weakly-referencable object, or if callback is not callable, None, or + NULL, this will return NULL and raise TypeError. + """ + w_weakref = space.gettypeobject(W_Weakref.typedef) + return space.call_function(w_weakref, w_obj, w_callback) + + at cpython_api([PyObject], PyObject, borrowed=True) +def PyWeakref_GetObject(space, w_ref): + """Return the referenced object from a weak reference, ref. If the referent is + no longer live, returns None. + """ + register_container(space, w_ref) + return space.call_function(w_ref) + From afa at codespeak.net Tue Apr 27 10:36:09 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 27 Apr 2010 10:36:09 +0200 (CEST) Subject: [pypy-svn] r74101 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100427083609.E30DC282BDB@codespeak.net> Author: afa Date: Tue Apr 27 10:36:08 2010 New Revision: 74101 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Log: reorder functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyerrors.py Tue Apr 27 10:36:08 2010 @@ -21,34 +21,10 @@ message = rffi.charp2str(message_ptr) PyErr_SetObject(space, w_type, space.wrap(message)) - at cpython_api([], lltype.Void) -def PyErr_BadArgument(space): - """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where - message indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use.""" - raise OperationError(space.w_TypeError, - space.wrap("bad argument type for built-in operation")) - - at cpython_api([PyObject], PyObject) -def PyErr_SetFromErrno(space, w_type): - """ - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable errno. It constructs a - tuple object whose first item is the integer errno value and whose - second item is the corresponding error message (gotten from strerror()), - and then calls PyErr_SetObject(type, object). On Unix, when the - errno value is EINTR, indicating an interrupted system call, - this calls PyErr_CheckSignals(), and if that set the error indicator, - leaves it set to that. The function always returns NULL, so a wrapper - function around a system call can write return PyErr_SetFromErrno(type); - when the system call returns an error. - Return value: always NULL.""" - # XXX Doesn't actually do anything with PyErr_CheckSignals. - errno = get_errno() - errno_w = space.wrap(errno) - message_w = space.wrap(os.strerror(errno)) - PyErr_SetObject(space, w_type, errno_w, message_w) - + at cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL) +def PyErr_SetNone(space, w_type): + """This is a shorthand for PyErr_SetObject(type, Py_None).""" + PyErr_SetObject(space, w_type, space.w_None) @cpython_api([], PyObject, borrowed=True) def PyErr_Occurred(space): @@ -99,9 +75,45 @@ Py_DecRef(space, w_value) @cpython_api([], lltype.Void) +def PyErr_BadArgument(space): + """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where + message indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use.""" + raise OperationError(space.w_TypeError, + space.wrap("bad argument type for built-in operation")) + + at cpython_api([], lltype.Void) def PyErr_BadInternalCall(space): raise OperationError(space.w_SystemError, space.wrap("Bad internal call!")) + at cpython_api([], PyObject, error=CANNOT_FAIL) +def PyErr_NoMemory(space): + """This is a shorthand for PyErr_SetNone(PyExc_MemoryError); it returns NULL + so an object allocation function can write return PyErr_NoMemory(); when it + runs out of memory. + Return value: always NULL.""" + PyErr_SetNone(space, space.w_MemoryError) + + at cpython_api([PyObject], PyObject) +def PyErr_SetFromErrno(space, w_type): + """ + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable errno. It constructs a + tuple object whose first item is the integer errno value and whose + second item is the corresponding error message (gotten from strerror()), + and then calls PyErr_SetObject(type, object). On Unix, when the + errno value is EINTR, indicating an interrupted system call, + this calls PyErr_CheckSignals(), and if that set the error indicator, + leaves it set to that. The function always returns NULL, so a wrapper + function around a system call can write return PyErr_SetFromErrno(type); + when the system call returns an error. + Return value: always NULL.""" + # XXX Doesn't actually do anything with PyErr_CheckSignals. + errno = get_errno() + errno_w = space.wrap(errno) + message_w = space.wrap(os.strerror(errno)) + PyErr_SetObject(space, w_type, errno_w, message_w) + @cpython_api([], rffi.INT_real, error=-1) def PyErr_CheckSignals(space): """ @@ -116,20 +128,6 @@ # XXX implement me return 0 - at cpython_api([], PyObject, error=CANNOT_FAIL) -def PyErr_NoMemory(space): - """This is a shorthand for PyErr_SetNone(PyExc_MemoryError); it returns NULL - so an object allocation function can write return PyErr_NoMemory(); when it - runs out of memory. - Return value: always NULL.""" - PyErr_SetNone(space, space.w_MemoryError) - - at cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL) -def PyErr_SetNone(space, w_type): - """This is a shorthand for PyErr_SetObject(type, Py_None).""" - PyErr_SetObject(space, w_type, space.w_None) - - @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyErr_GivenExceptionMatches(space, w_given, w_exc): """Return true if the given exception matches the exception in exc. If From afa at codespeak.net Tue Apr 27 10:50:44 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 27 Apr 2010 10:50:44 +0200 (CEST) Subject: [pypy-svn] r74102 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100427085044.47485282B9D@codespeak.net> Author: afa Date: Tue Apr 27 10:50:42 2010 New Revision: 74102 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix PyEval_SaveThread, which crashed when the thread module is compiled, but no thread has been started Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pystate.py Tue Apr 27 10:50:42 2010 @@ -13,9 +13,8 @@ previous thread state (which is not NULL except in PyPy). If the lock has been created, the current thread must have acquired it. (This function is available even when thread support is disabled at compile time.)""" - if space.config.objspace.usemodules.thread: - from pypy.module.thread.gil import before_external_call - before_external_call() + if rffi.aroundstate.before: + rffi.aroundstate.before() return lltype.nullptr(PyThreadState.TO) @cpython_api([PyThreadState], lltype.Void) @@ -25,9 +24,8 @@ NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. (This function is available even when thread support is disabled at compile time.)""" - if space.config.objspace.usemodules.thread: - from pypy.module.thread.gil import after_external_call - after_external_call() + if rffi.aroundstate.after: + rffi.aroundstate.after() @cpython_api([], lltype.Void) def PyEval_InitThreads(space): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Tue Apr 27 10:50:42 2010 @@ -40,7 +40,7 @@ class AppTestApi: def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cpyext']) + cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) from pypy.rlib.libffi import get_libc_name cls.w_libc = cls.space.wrap(get_libc_name()) @@ -129,7 +129,7 @@ class AppTestCpythonExtensionBase: def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cpyext']) + cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") def import_module(self, name, init=None, body='', load_it=True): From afa at codespeak.net Tue Apr 27 10:59:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 27 Apr 2010 10:59:46 +0200 (CEST) Subject: [pypy-svn] r74103 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100427085946.80576282B9D@codespeak.net> Author: afa Date: Tue Apr 27 10:59:44 2010 New Revision: 74103 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/conftest.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix tests: we should have only one object space, and ensure that the lock object on sys.stdout is created before we count allocations. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/conftest.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/conftest.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/conftest.py Tue Apr 27 10:59:44 2010 @@ -12,7 +12,7 @@ return super(Directory, self).collect() def pytest_funcarg__space(request): - return gettestobjspace(usemodules=['cpyext']) + return gettestobjspace(usemodules=['cpyext', 'thread']) def pytest_funcarg__api(request): return request.cls.api Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_api.py Tue Apr 27 10:59:44 2010 @@ -17,7 +17,7 @@ class BaseApiTest: def setup_class(cls): - cls.space = gettestobjspace(usemodules=['cpyext']) + cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") class CAPI: def __getattr__(self, name): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Tue Apr 27 10:59:44 2010 @@ -195,6 +195,10 @@ def setup_method(self, func): self.w_import_module = self.space.wrap(self.import_module) self.w_import_extension = self.space.wrap(self.import_extension) + + # create the file lock before we count allocations + self.space.call_method(self.space.sys.get("stdout"), "flush") + freeze_refcnts(self) #self.check_and_print_leaks() From arigo at codespeak.net Tue Apr 27 11:33:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 11:33:24 +0200 (CEST) Subject: [pypy-svn] r74104 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427093324.D4DFD282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 11:33:23 2010 New Revision: 74104 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Log: Remove the hacking of the formatted assembler, and print directly in a simpler format. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Tue Apr 27 11:33:23 2010 @@ -1,5 +1,6 @@ import py from pypy.objspace.flow.model import Constant +from pypy.rpython.lltypesystem import lltype from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr from pypy.jit.metainterp.history import AbstractDescr @@ -14,6 +15,9 @@ if isinstance(x, Register): return '%%%s%d' % (x.kind[0], x.index) # e.g. %i1 or %r2 or %f3 elif isinstance(x, Constant): + if (isinstance(x.concretetype, lltype.Ptr) and + isinstance(x.concretetype.TO, lltype.Struct)): + return '$<* struct %s>' % (x.concretetype.TO._name,) return '$%r' % (x.value,) elif isinstance(x, TLabel): return getlabelname(x) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 11:33:23 2010 @@ -53,32 +53,11 @@ self.assert_format(ssarepr, expected) def assert_format(self, ssarepr, expected): - def replace_struct(s): - """ Replace all $<* struct ...> with $STRUCT - """ - while True: - i = s.find('$<* struct') - if i == -1: - return s - count = 1 - start = i - i += 2 - while True: - if s[i] == '<': - count += 1 - if s[i] == '>': - count -= 1 - if count == 0: - break - i += 1 - s = s[:start] + '$STRUCT' + s[i + 1:] - asm = format_assembler(ssarepr) expected = str(py.code.Source(expected)).strip() + '\n' asmlines = asm.split("\n") explines = expected.split("\n") for asm, exp in zip(asmlines, explines): - asm = replace_struct(asm) if asm != exp: print print "Got: " + asm @@ -258,10 +237,10 @@ goto_if_exception L1 int_return $3 L1: - goto_if_exception_mismatch $STRUCT, L2 + goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $1 L2: - goto_if_exception_mismatch $STRUCT, L3 + goto_if_exception_mismatch $<* struct object_vtable>, L3 int_return $2 L3: reraise Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_format.py Tue Apr 27 11:33:23 2010 @@ -33,14 +33,28 @@ """ assert asm == str(py.code.Source(expected)).strip() + '\n' +def test_format_assembler_const_struct(): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + s = lltype.malloc(S) + s.x = 123 + ssarepr = SSARepr("test") + ssarepr.insns = [ + ('foobar', Constant(s, lltype.typeOf(s))), + ] + asm = format_assembler(ssarepr) + expected = """ + foobar $<* struct S> + """ + assert asm == str(py.code.Source(expected)).strip() + '\n' + def test_format_assembler_loop(): ssarepr = SSARepr("test") i0, i1 = Register('int', 0), Register('int', 1) ssarepr.insns = [ (Label('L1'),), - ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0)), + ('goto_if_not_int_gt', TLabel('L2'), i0, Constant(0, lltype.Signed)), ('int_add', i1, i0, i1), - ('int_sub', i0, Constant(1), i0), + ('int_sub', i0, Constant(1, lltype.Signed), i0), ('goto', TLabel('L1')), (Label('L2'),), ('int_return', i1), From arigo at codespeak.net Tue Apr 27 11:40:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 11:40:59 +0200 (CEST) Subject: [pypy-svn] r74105 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100427094059.9013F282BDB@codespeak.net> Author: arigo Date: Tue Apr 27 11:40:58 2010 New Revision: 74105 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Log: Uh? Here I really meant "== ()", and not just "is some empty container". Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 11:40:58 2010 @@ -172,9 +172,10 @@ self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) for link in block.exits[1:]: - if (link.exitcase is Exception and (not link.target.operations) + if (link.exitcase is Exception and link.target.operations == () and len(link.target.inputargs) == 2): - # default exit-by-exception block + # default exit-by-exception block, if the link is going + # directly to the except block. self.emitline("reraise") else: self.emitline('goto_if_exception_mismatch', From arigo at codespeak.net Tue Apr 27 13:18:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 13:18:49 +0200 (CEST) Subject: [pypy-svn] r74106 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100427111849.C8794282BDB@codespeak.net> Author: arigo Date: Tue Apr 27 13:18:48 2010 New Revision: 74106 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Log: Preserve the tuple-ness of () in block.operations. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Tue Apr 27 13:18:48 2010 @@ -24,6 +24,8 @@ def transform(self, graph): self.graph = graph for block in graph.iterblocks(): + if block.operations == (): + continue rename = {} newoperations = [] if block.exitswitch == c_last_exception: From arigo at codespeak.net Tue Apr 27 13:19:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 13:19:55 +0200 (CEST) Subject: [pypy-svn] r74107 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427111955.97169282BDB@codespeak.net> Author: arigo Date: Tue Apr 27 13:19:54 2010 New Revision: 74107 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Finish the bytecode format for exceptions (it was missing 'last_exception' and 'last_exc_value' support). Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 13:19:54 2010 @@ -127,6 +127,17 @@ self.insert_renamings(link) self.make_bytecode_block(link.target) + def make_exception_link(self, link): + # Like make_link(), but also introduces the 'last_exception' and + # 'last_exc_value' as variables if needed + assert link.last_exception is not None + assert link.last_exc_value is not None + if link.last_exception in link.args: + self.emitline("last_exception", self.getcolor(link.last_exception)) + if link.last_exc_value in link.args: + self.emitline("last_exc_value", self.getcolor(link.last_exc_value)) + self.make_link(link) + def insert_exits(self, block): if len(block.exits) == 1: # A single link, fall-through @@ -168,22 +179,29 @@ # exc_2 case # reraise assert block.exits[0].exitcase is None # is this always True? - self.emitline('goto_if_exception', TLabel(block.exits[0])) + self.emitline('catch', TLabel(block.exits[0])) self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) for link in block.exits[1:]: - if (link.exitcase is Exception and link.target.operations == () - and len(link.target.inputargs) == 2): - # default exit-by-exception block, if the link is going - # directly to the except block. - self.emitline("reraise") - else: - self.emitline('goto_if_exception_mismatch', - Constant(link.llexitcase, - lltype.typeOf(link.llexitcase)), - TLabel(link)) - self.make_link(link) - self.emitline(Label(link)) + if link.exitcase is Exception: + # this link captures all exceptions + if (link.target.operations == () + and len(link.target.inputargs) == 2): + # the link is going directly to the except block + self.emitline("reraise") + else: + self.make_exception_link(link) + break + self.emitline('goto_if_exception_mismatch', + Constant(link.llexitcase, + lltype.typeOf(link.llexitcase)), + TLabel(link)) + self.make_exception_link(link) + self.emitline(Label(link)) + else: + # no link captures all exceptions, so we have to put a reraise + # for the other exceptions + self.emitline("reraise") else: # A switch. # Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 13:19:54 2010 @@ -21,6 +21,16 @@ self.num_colors += 1 return self.seen[v] +class FakeDescr(AbstractDescr): + def __repr__(self): + return '' + +class FakeCPU: + def calldescrof(self, FUNC, ARGS, RESULT): + return FakeDescr() + def fielddescrof(self, STRUCT, name): + return FakeDescr() + def fake_regallocs(): return {'int': FakeRegAlloc(), 'ref': FakeRegAlloc(), @@ -48,7 +58,7 @@ def encoding_test(self, func, args, expected, transform=False): graphs = self.make_graphs(func, args) if transform: - transform_graph(graphs[0]) + transform_graph(graphs[0], FakeCPU()) ssarepr = flatten_graph(graphs[0], fake_regallocs()) self.assert_format(ssarepr, expected) @@ -234,7 +244,7 @@ self.encoding_test(f, [65], """ direct_call $<* fn g>, %i0 - goto_if_exception L1 + catch L1 int_return $3 L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 @@ -245,3 +255,33 @@ L3: reraise """) + + def test_exc_exitswitch_2(self): + class FooError(Exception): + pass + def g(i): + FooError().num = 1 + FooError().num = 2 + def f(i): + try: + g(i) + except FooError, e: + return e.num + except Exception: + return 3 + else: + return 4 + + self.encoding_test(f, [65], """ + residual_call_ir_v $<* fn g>, , I[%i0], R[] + catch L1 + int_return $4 + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 + last_exc_value %r0 + ref_copy %r0, %r1 + getfield_gc_i %r1, , %i1 + int_return %i1 + L2: + int_return $3 + """, transform=True) From arigo at codespeak.net Tue Apr 27 13:50:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 13:50:02 +0200 (CEST) Subject: [pypy-svn] r74108 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427115002.948A5282BDB@codespeak.net> Author: arigo Date: Tue Apr 27 13:50:00 2010 New Revision: 74108 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Tests and fixes. Also generate "try_catch" just before the instruction that can actually raise, instead of just after. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 13:50:00 2010 @@ -116,7 +116,7 @@ self.emitline("%s_return" % kind, self.getcolor(args[0])) elif len(args) == 2: # exception block, raising an exception from a function - xxx + self.emitline("raise", self.getcolor(args[1])) else: raise Exception("?") @@ -145,6 +145,35 @@ assert link.exitcase is None self.make_link(link) # + elif block.exitswitch is c_last_exception: + # An exception block. See test_exc_exitswitch in test_flatten.py + # for an example of what kind of code this makes. + assert block.exits[0].exitcase is None # is this always True? + self.emitlinebefore('_call', 'try_catch', TLabel(block.exits[0])) + self.make_link(block.exits[0]) + self.emitline(Label(block.exits[0])) + for link in block.exits[1:]: + if link.exitcase is Exception: + # this link captures all exceptions + if (link.target.operations == () + and link.args == [link.last_exception, + link.last_exc_value]): + # the link is going directly to the except block + self.emitline("reraise") + else: + self.make_exception_link(link) + break + self.emitline('goto_if_exception_mismatch', + Constant(link.llexitcase, + lltype.typeOf(link.llexitcase)), + TLabel(link)) + self.make_exception_link(link) + self.emitline(Label(link)) + else: + # no link captures all exceptions, so we have to put a reraise + # for the other exceptions + self.emitline("reraise") + # elif len(block.exits) == 2 and ( isinstance(block.exitswitch, tuple) or block.exitswitch.concretetype == lltype.Bool): @@ -169,39 +198,6 @@ self.emitline(Label(linkfalse)) self.make_link(linkfalse) # - elif block.exitswitch is c_last_exception: - # An exception block. Would create something like: - # if exception jump first check - # defaultcase - # if not exc_1 jmp next check - # exc_1 case - # if not exc_2 jmp next check - # exc_2 case - # reraise - assert block.exits[0].exitcase is None # is this always True? - self.emitline('catch', TLabel(block.exits[0])) - self.make_link(block.exits[0]) - self.emitline(Label(block.exits[0])) - for link in block.exits[1:]: - if link.exitcase is Exception: - # this link captures all exceptions - if (link.target.operations == () - and len(link.target.inputargs) == 2): - # the link is going directly to the except block - self.emitline("reraise") - else: - self.make_exception_link(link) - break - self.emitline('goto_if_exception_mismatch', - Constant(link.llexitcase, - lltype.typeOf(link.llexitcase)), - TLabel(link)) - self.make_exception_link(link) - self.emitline(Label(link)) - else: - # no link captures all exceptions, so we have to put a reraise - # for the other exceptions - self.emitline("reraise") else: # A switch. # @@ -278,6 +274,13 @@ def emitline(self, *line): self.ssarepr.insns.append(line) + def emitlinebefore(self, name_extract, *line): + assert name_extract in self.ssarepr.insns[-1][0], ( + "emitlinebefore: the %s operation should be inserted before %s," + " but it does not look like a %r operation" % ( + line, self.ssarepr.insns[-1], name_extract)) + self.ssarepr.insns.insert(-1, line) + def flatten_list(self, arglist): args = [] for v in arglist: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/regalloc.py Tue Apr 27 13:50:00 2010 @@ -83,6 +83,10 @@ # than the start, given that we resume execution from the # middle during blackholing. for link in block.exits: + if link.last_exception is not None: + self._depgraph.add_node(link.last_exception) + if link.last_exc_value is not None: + self._depgraph.add_node(link.last_exc_value) for i, v in enumerate(link.args): self._try_coalesce(v, link.target.inputargs[i]) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 13:50:00 2010 @@ -243,8 +243,8 @@ return 3 self.encoding_test(f, [65], """ + try_catch L1 direct_call $<* fn g>, %i0 - catch L1 int_return $3 L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 @@ -273,8 +273,8 @@ return 4 self.encoding_test(f, [65], """ + try_catch L1 residual_call_ir_v $<* fn g>, , I[%i0], R[] - catch L1 int_return $4 L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 @@ -285,3 +285,31 @@ L2: int_return $3 """, transform=True) + + def test_exc_raise_1(self): + class FooError(Exception): + pass + fooerror = FooError() + def f(i): + raise fooerror + + self.encoding_test(f, [65], """ + raise $<* struct object> + """) + + def test_exc_raise_2(self): + def g(i): + pass + def f(i): + try: + g(i) + except Exception: + raise KeyError + + self.encoding_test(f, [65], """ + try_catch L1 + direct_call $<* fn g>, %i0 + void_return + L1: + raise $<* struct object> + """) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Tue Apr 27 13:50:00 2010 @@ -5,7 +5,8 @@ from pypy.jit.codewriter.format import format_assembler from pypy.objspace.flow.model import Variable, Constant, SpaceOperation from pypy.objspace.flow.model import FunctionGraph, Block, Link -from pypy.rpython.lltypesystem import lltype +from pypy.objspace.flow.model import c_last_exception +from pypy.rpython.lltypesystem import lltype, rclass class TestRegAlloc: @@ -145,3 +146,33 @@ rescall I[%i0, %i1], %i0 int_return %i0 """) + + def test_regalloc_exitswitch_2(self): + v1 = Variable(); v1.concretetype = rclass.CLASSTYPE + v2 = Variable(); v2.concretetype = rclass.CLASSTYPE + v3 = Variable(); v3.concretetype = rclass.CLASSTYPE + v4 = Variable(); v4.concretetype = rclass.CLASSTYPE + block = Block([]) + block.operations = [ + SpaceOperation('res_call', [], v1), + ] + graph = FunctionGraph('f', block, v4) + exclink = Link([v2], graph.returnblock) + exclink.llexitcase = 123 # normally an exception class + exclink.last_exception = v2 + exclink.last_exc_value = "unused" + block.exitswitch = c_last_exception + block.closeblock(Link([v1], graph.returnblock), + exclink) + # + self.check_assembler(graph, """ + try_catch L1 + res_call %i0 + int_return %i0 + L1: + goto_if_exception_mismatch $123, L2 + last_exception %i0 + int_return %i0 + L2: + reraise + """) From arigo at codespeak.net Tue Apr 27 14:22:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 14:22:46 +0200 (CEST) Subject: [pypy-svn] r74109 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100427122246.98C52282BDB@codespeak.net> Author: arigo Date: Tue Apr 27 14:22:45 2010 New Revision: 74109 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Log: Start working on exceptions in the blachole interp. Reverted a change of r74108: it is after all easier to have the 'catch_exception' operation after the operation that can raise rather than before. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 14:22:45 2010 @@ -149,7 +149,7 @@ # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. assert block.exits[0].exitcase is None # is this always True? - self.emitlinebefore('_call', 'try_catch', TLabel(block.exits[0])) + self.emitline('catch_exception', TLabel(block.exits[0])) self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) for link in block.exits[1:]: @@ -274,13 +274,6 @@ def emitline(self, *line): self.ssarepr.insns.append(line) - def emitlinebefore(self, name_extract, *line): - assert name_extract in self.ssarepr.insns[-1][0], ( - "emitlinebefore: the %s operation should be inserted before %s," - " but it does not look like a %r operation" % ( - line, self.ssarepr.insns[-1], name_extract)) - self.ssarepr.insns.insert(-1, line) - def flatten_list(self, arglist): args = [] for v in arglist: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 14:22:45 2010 @@ -243,8 +243,8 @@ return 3 self.encoding_test(f, [65], """ - try_catch L1 direct_call $<* fn g>, %i0 + catch_exception L1 int_return $3 L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 @@ -273,8 +273,8 @@ return 4 self.encoding_test(f, [65], """ - try_catch L1 residual_call_ir_v $<* fn g>, , I[%i0], R[] + catch_exception L1 int_return $4 L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 @@ -307,8 +307,8 @@ raise KeyError self.encoding_test(f, [65], """ - try_catch L1 direct_call $<* fn g>, %i0 + catch_exception L1 void_return L1: raise $<* struct object> Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Tue Apr 27 14:22:45 2010 @@ -166,8 +166,8 @@ exclink) # self.check_assembler(graph, """ - try_catch L1 res_call %i0 + catch_exception L1 int_return %i0 L1: goto_if_exception_mismatch $123, L2 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue Apr 27 14:22:45 2010 @@ -46,6 +46,7 @@ for key, value in insns.items(): assert self._insns[value] is None self._insns[value] = key + self.op_catch_exception = insns.get('catch_exception/L', -1) # all_funcs = [] for key in self._insns: @@ -123,7 +124,14 @@ else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) - result = unboundmethod(self, *args) + # call the method opimpl_xxx() + try: + result = unboundmethod(self, *args) + except Exception: + if resulttype == 'i' or resulttype == 'r' or resulttype == 'f': + position += 1 + self.exception_pc = position + raise if resulttype == 'i': # argcode should be 'i' too assert argcodes[next_argcode] == 'i' @@ -172,9 +180,11 @@ class BlackholeInterpreter(object): def __init__(self, builder): - self.cpu = builder.cpu - self.dispatch_loop = builder.dispatch_loop - self.descrs = builder.descrs + self.cpu = builder.cpu + self.dispatch_loop = builder.dispatch_loop + self.descrs = builder.descrs + self.op_catch_exception = builder.op_catch_exception + # if we_are_translated(): default_i = 0 default_r = lltype.nullptr(llmemory.GCREF.TO) @@ -195,10 +205,28 @@ self.copy_constants(self.registers_r, jitcode.constants_r) self.copy_constants(self.registers_f, jitcode.constants_f) code = jitcode.code - try: - self.dispatch_loop(self, code, position) - except LeaveFrame: - pass + while True: + try: + self.dispatch_loop(self, code, position) + except LeaveFrame: + return + #except JitException: + # ... + except Exception, e: + position = self.handle_exception_in_frame(e, code) + + def handle_exception_in_frame(self, e, code): + # This frame raises an exception. First try to see if + # the exception is handled in the frame itself. + position = self.exception_pc # <-- just after the insn that raised + opcode = ord(code[position]) + if opcode != self.op_catch_exception: + raise e # no 'catch_exception' insn follows: just reraise + else: + # else store the exception on 'self', and jump to the handler + self.exception_last_value = e + target = ord(code[position+1]) | (ord(code[position+2])<<8) + return target # XXX must be specialized # XXX the real performance impact of the following loop is unclear @@ -307,6 +335,13 @@ except KeyError: return pc + @arguments("L") + def opimpl_catch_exception(self, target): + """This is a no-op when run normally. When an exception occurs + and the instruction that raised is immediately followed by a + catch_exception, then the code in handle_exception_in_frame() + will capture the exception and jump to 'target'.""" + # ---------- # the following operations are directly implemented by the backend Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Tue Apr 27 14:22:45 2010 @@ -1,17 +1,38 @@ from pypy.rlib.jit import JitDriver from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.jit.metainterp.blackhole import BlackholeInterpreter +from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder from pypy.jit.codewriter.assembler import JitCode +from pypy.rpython.lltypesystem import lltype, llmemory +class FakeCodeWriter: + pass +class FakeAssembler: + pass +class FakeCPU: + def bh_call_i(self, func, calldescr, args_i, args_r, args_f): + assert func == 321 + assert calldescr == "" + if args_i[0] < 0: + raise KeyError + return args_i[0] * 2 + +def getblackholeinterp(insns, descrs=[]): + cw = FakeCodeWriter() + cw.cpu = FakeCPU() + cw.assembler = FakeAssembler() + cw.assembler.insns = insns + cw.assembler.descrs = descrs + builder = BlackholeInterpBuilder(cw) + return builder.acquire_interp() + def test_simple(): jitcode = JitCode("test") jitcode.setup("\x00\x00\x01\x02" "\x01\x02", []) - blackholeinterp = BlackholeInterpreter() - blackholeinterp.setup_insns({'int_add/iii': 0, - 'int_return/i': 1}) + blackholeinterp = getblackholeinterp({'int_add/iii': 0, + 'int_return/i': 1}) blackholeinterp.setarg_i(0, 40) blackholeinterp.setarg_i(1, 2) blackholeinterp.run(jitcode, 0) @@ -22,9 +43,8 @@ jitcode.setup("\x00\x30\x01\x02" "\x01\x02", []) - blackholeinterp = BlackholeInterpreter() - blackholeinterp.setup_insns({'int_sub/cii': 0, - 'int_return/i': 1}) + blackholeinterp = getblackholeinterp({'int_sub/cii': 0, + 'int_return/i': 1}) blackholeinterp.setarg_i(1, 6) blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 42 @@ -34,9 +54,8 @@ jitcode.setup("\x00\xFD\x01\x02" "\x01\x02", [666, 666, 10042, 666]) - blackholeinterp = BlackholeInterpreter() - blackholeinterp.setup_insns({'int_sub/iii': 0, - 'int_return/i': 1}) + blackholeinterp = getblackholeinterp({'int_sub/iii': 0, + 'int_return/i': 1}) blackholeinterp.setarg_i(1, 10000) blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 42 @@ -49,17 +68,38 @@ "\x03\x00\x00" # goto L1 "\x04\x17", # L2: int_return %i1 []) - blackholeinterp = BlackholeInterpreter() - blackholeinterp.setup_insns({'goto_if_not_int_gt/Lic': 0, - 'int_add/iii': 1, - 'int_sub/ici': 2, - 'goto/L': 3, - 'int_return/i': 4}) + blackholeinterp = getblackholeinterp({'goto_if_not_int_gt/Lic': 0, + 'int_add/iii': 1, + 'int_sub/ici': 2, + 'goto/L': 3, + 'int_return/i': 4}) blackholeinterp.setarg_i(0x16, 6) # %i0 blackholeinterp.setarg_i(0x17, 100) # %i1 blackholeinterp.run(jitcode, 0) assert blackholeinterp.result_i == 100+6+5+4+3 +def test_simple_exception(): + jitcode = JitCode("test") + jitcode.setup( # residual_call_ir_i $<* fn g>, , I[%i9], R[], %i8 + "\x01\xFF\x00\x00\x01\x09\x00\x08" + "\x00\x0D\x00" # catch_exception L1 + "\x02\x08" # int_return %i8 + "\x03\x2A", # L1: int_return $42 + [321]) # <-- address of the function g + blackholeinterp = getblackholeinterp({'catch_exception/L': 0, + 'residual_call_ir_i/idIRi': 1, + 'int_return/i': 2, + 'int_return/c': 3}, + [""]) + # + blackholeinterp.setarg_i(0x9, 100) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 200 + # + blackholeinterp.setarg_i(0x9, -100) + blackholeinterp.run(jitcode, 0) + assert blackholeinterp.result_i == 42 + # ____________________________________________________________ class BlackholeTests(object): From arigo at codespeak.net Tue Apr 27 15:31:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 15:31:27 +0200 (CEST) Subject: [pypy-svn] r74110 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20100427133127.2770C282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 15:31:25 2010 New Revision: 74110 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Log: Correctly catch the exceptions, at least when not translated for now. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Tue Apr 27 15:31:25 2010 @@ -1352,9 +1352,7 @@ def do_call_pushfloat(x): _call_args_f.append(x) -def _do_call_common(f, err_result=None): - global _last_exception - assert _last_exception is None, "exception left behind" +def _do_call_common(f): ptr = llmemory.cast_int_to_adr(f).ptr FUNC = lltype.typeOf(ptr).TO ARGS = FUNC.ARGS @@ -1363,30 +1361,27 @@ del _call_args_r[:] del _call_args_f[:] assert len(ARGS) == len(args) - try: - if hasattr(ptr._obj, 'graph'): - llinterp = _llinterp # it's a global set here by CPU.__init__() - result = llinterp.eval_graph(ptr._obj.graph, args) - else: - result = ptr._obj._callable(*args) - except LLException, e: - _last_exception = e - result = err_result + if hasattr(ptr._obj, 'graph'): + llinterp = _llinterp # it's a global set here by CPU.__init__() + result = llinterp.eval_graph(ptr._obj.graph, args) + # ^^^ may raise, in which case we get an LLException + else: + result = ptr._obj._callable(*args) return result def do_call_void(f): _do_call_common(f) def do_call_int(f): - x = _do_call_common(f, 0) + x = _do_call_common(f) return cast_to_int(x) def do_call_float(f): - x = _do_call_common(f, 0) + x = _do_call_common(f) return cast_to_float(x) def do_call_ptr(f): - x = _do_call_common(f, lltype.nullptr(llmemory.GCREF.TO)) + x = _do_call_common(f) return cast_to_ptr(x) def cast_call_args(ARGS, args_i, args_r, args_f): @@ -1432,7 +1427,7 @@ result = llinterp.eval_graph(mymethod.graph, myargs) else: result = meth(*args) - except LLException, e: + except XXX-LLException, e: _last_exception = e result = get_err_result_for_type(mymethod._TYPE.RESULT) return result Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Tue Apr 27 15:31:25 2010 @@ -270,12 +270,6 @@ # ---------- - def get_exception(self): - return self.cast_adr_to_int(llimpl.get_exception()) - - def get_exc_value(self): - return llimpl.get_exc_value() - def clear_exception(self): llimpl.clear_exception() Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue Apr 27 15:31:25 2010 @@ -2,8 +2,9 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint from pypy.rlib.objectmodel import we_are_translated from pypy.tool.sourcetools import func_with_new_name -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.llinterp import LLException from pypy.jit.codewriter.flatten import SwitchDictDescr @@ -29,6 +30,7 @@ class BlackholeInterpBuilder(object): + verbose = True def __init__(self, codewriter): self.cpu = codewriter.cpu @@ -124,14 +126,27 @@ else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) + + if verbose and not we_are_translated(): + print '\t', name, list(args), + # call the method opimpl_xxx() try: result = unboundmethod(self, *args) - except Exception: + except Exception, e: + if verbose and not we_are_translated(): + print '-> %s!' % (e.__class__.__name__,) if resulttype == 'i' or resulttype == 'r' or resulttype == 'f': position += 1 self.exception_pc = position raise + + if verbose and not we_are_translated(): + if result is None: + print + else: + print '->', result + if resulttype == 'i': # argcode should be 'i' too assert argcodes[next_argcode] == 'i' @@ -161,6 +176,7 @@ assert next_argcode == len(argcodes) return position # + verbose = self.verbose unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name) argtypes = unrolling_iterable(unboundmethod.argtypes) resulttype = unboundmethod.resulttype @@ -213,6 +229,9 @@ #except JitException: # ... except Exception, e: + if not we_are_translated(): + if not isinstance(e, LLException): + raise position = self.handle_exception_in_frame(e, code) def handle_exception_in_frame(self, e, code): @@ -224,6 +243,8 @@ raise e # no 'catch_exception' insn follows: just reraise else: # else store the exception on 'self', and jump to the handler + if not we_are_translated(): # get the lltyped exception + e = e.args[1] # object out of the LLException self.exception_last_value = e target = ord(code[position+1]) | (ord(code[position+2])<<8) return target @@ -342,6 +363,36 @@ catch_exception, then the code in handle_exception_in_frame() will capture the exception and jump to 'target'.""" + @arguments("i", "L", "pc", returns="L") + def opimpl_goto_if_exception_mismatch(self, vtable, target, pc): + adr = llmemory.cast_int_to_adr(vtable) + bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) + real_instance = self.exception_last_value + assert real_instance + if rclass.ll_issubclass(real_instance.typeptr, bounding_class): + return pc + else: + return target + + @arguments(returns="i") + def opimpl_last_exception(self): + real_instance = self.exception_last_value + assert real_instance + adr = llmemory.cast_ptr_to_adr(real_instance.typeptr) + return llmemory.cast_adr_to_int(adr) + + @arguments(returns="r") + def opimpl_last_exc_value(self): + real_instance = self.exception_last_value + assert real_instance + return lltype.cast_opaque_ptr(llmemory.GCREF, real_instance) + + @arguments() + def opimpl_reraise(self): + real_instance = self.exception_last_value + assert real_instance + raise real_instance + # ---------- # the following operations are directly implemented by the backend Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Tue Apr 27 15:31:25 2010 @@ -3,6 +3,7 @@ from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder from pypy.jit.codewriter.assembler import JitCode from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.llinterp import LLException class FakeCodeWriter: @@ -14,7 +15,7 @@ assert func == 321 assert calldescr == "" if args_i[0] < 0: - raise KeyError + raise LLException("etype", "evalue") return args_i[0] * 2 def getblackholeinterp(insns, descrs=[]): From arigo at codespeak.net Tue Apr 27 15:56:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 15:56:31 +0200 (CEST) Subject: [pypy-svn] r74111 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427135631.0F2B7282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 15:56:30 2010 New Revision: 74111 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Add the operation 'goto_if_not_int_is_true'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Tue Apr 27 15:56:30 2010 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.jit.metainterp.history import getkind -from pypy.objspace.flow.model import SpaceOperation, Variable, c_last_exception +from pypy.objspace.flow.model import SpaceOperation, Variable, Constant +from pypy.objspace.flow.model import c_last_exception from pypy.jit.codewriter.flatten import ListOfKind @@ -84,11 +85,25 @@ return False # variable is also used in cur block if v is op.result: if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', - 'int_gt', 'int_ge'): + 'int_gt', 'int_ge', 'int_is_true'): return False # not a supported operation # ok! optimize this case block.operations.remove(op) - block.exitswitch = (op.opname,) + tuple(op.args) + opname = op.opname + args = op.args + if op.opname in ('int_ne', 'int_eq'): + if isinstance(args[0], Constant): + args = args[::-1] + if isinstance(args[1], Constant) and args[1].value == 0: + if opname == 'int_eq': + # must invert the two exit links + link = block.exits[0] + link.llexitcase = link.exitcase = not link.exitcase + link = block.exits[1] + link.llexitcase = link.exitcase = not link.exitcase + opname = 'int_is_true' + args = [args[0]] + block.exitswitch = (opname,) + tuple(args) return True return False Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 15:56:30 2010 @@ -8,6 +8,7 @@ from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.translator.unsimplify import varoftype +from pypy.rlib.rarithmetic import ovfcheck class FakeRegAlloc: @@ -243,17 +244,17 @@ return 3 self.encoding_test(f, [65], """ - direct_call $<* fn g>, %i0 - catch_exception L1 - int_return $3 - L1: - goto_if_exception_mismatch $<* struct object_vtable>, L2 - int_return $1 - L2: - goto_if_exception_mismatch $<* struct object_vtable>, L3 - int_return $2 - L3: - reraise + direct_call $<* fn g>, %i0 + catch_exception L1 + int_return $3 + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 + int_return $1 + L2: + goto_if_exception_mismatch $<* struct object_vtable>, L3 + int_return $2 + L3: + reraise """) def test_exc_exitswitch_2(self): @@ -273,17 +274,17 @@ return 4 self.encoding_test(f, [65], """ - residual_call_ir_v $<* fn g>, , I[%i0], R[] - catch_exception L1 - int_return $4 - L1: - goto_if_exception_mismatch $<* struct object_vtable>, L2 - last_exc_value %r0 - ref_copy %r0, %r1 - getfield_gc_i %r1, , %i1 - int_return %i1 - L2: - int_return $3 + residual_call_ir_v $<* fn g>, , I[%i0], R[] + catch_exception L1 + int_return $4 + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 + last_exc_value %r0 + ref_copy %r0, %r1 + getfield_gc_i %r1, , %i1 + int_return %i1 + L2: + int_return $3 """, transform=True) def test_exc_raise_1(self): @@ -307,9 +308,41 @@ raise KeyError self.encoding_test(f, [65], """ - direct_call $<* fn g>, %i0 - catch_exception L1 - void_return - L1: - raise $<* struct object> + direct_call $<* fn g>, %i0 + catch_exception L1 + void_return + L1: + raise $<* struct object> """) + + def test_goto_if_not_int_is_true(self): + def f(i): + return not i + + self.encoding_test(f, [7], """ + goto_if_not_int_is_true L1, %i0 + int_return $False + L1: + int_return $True + """, transform=True) + + def test_int_floordiv_ovf_zer(self): + py.test.skip("in-progress") + def f(i, j): + try: + return ovfcheck(i // j) + except OverflowError: + return 42 + except ZeroDivisionError: + return -42 + + self.encoding_test(f, [7, 2], """ + goto_if_not_int_is_true L1, %i1 + goto_if_div_overflow L2, %i0, %i1 + int_floordiv %i0, %i1, %i0 + int_return %i0 + L1: + int_return $-42 + L2: + int_return $42 + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Tue Apr 27 15:56:30 2010 @@ -14,6 +14,8 @@ class FakeLink: args = [] + def __init__(self, exitcase): + self.exitcase = self.llexitcase = exitcase def test_optimize_goto_if_not(): v1 = Variable() @@ -24,7 +26,7 @@ block = Block([v1, v2]) block.operations = [sp1, SpaceOperation('int_gt', [v1, v2], v3), sp2] block.exitswitch = v3 - block.exits = exits = [FakeLink(), FakeLink()] + block.exits = exits = [FakeLink(False), FakeLink(True)] res = Transformer().optimize_goto_if_not(block) assert res == True assert block.operations == [sp1, sp2] @@ -35,7 +37,7 @@ v1 = Variable(); v1.concretetype = lltype.Bool block = Block([v1]) block.exitswitch = v1 - block.exits = [FakeLink(), FakeLink()] + block.exits = [FakeLink(False), FakeLink(True)] assert not Transformer().optimize_goto_if_not(block) def test_optimize_goto_if_not__exit(): @@ -45,7 +47,7 @@ block = Block([v1, v2]) block.operations = [SpaceOperation('int_gt', [v1, v2], v3)] block.exitswitch = v3 - block.exits = [FakeLink(), FakeLink()] + block.exits = [FakeLink(False), FakeLink(True)] block.exits[1].args = [v3] assert not Transformer().optimize_goto_if_not(block) @@ -54,9 +56,35 @@ block = Block([]) block.operations = [SpaceOperation('foobar', [], v3)] block.exitswitch = v3 - block.exits = [FakeLink(), FakeLink()] + block.exits = [FakeLink(False), FakeLink(True)] assert not Transformer().optimize_goto_if_not(block) +def test_optimize_goto_if_not__int_ne(): + c0 = Constant(0, lltype.Signed) + v1 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + for linkcase1 in [False, True]: + linkcase2 = not linkcase1 + for op, args in [('int_ne', [v1, c0]), + ('int_ne', [c0, v1]), + ('int_eq', [v1, c0]), + ('int_eq', [c0, v1])]: + block = Block([v1]) + block.operations = [SpaceOperation(op, args, v3)] + block.exitswitch = v3 + block.exits = exits = [FakeLink(linkcase1), FakeLink(linkcase2)] + res = Transformer().optimize_goto_if_not(block) + assert res == True + assert block.operations == [] + assert block.exitswitch == ('int_is_true', v1) + assert block.exits == exits + if op == 'int_ne': + assert exits[0].exitcase == exits[0].llexitcase == linkcase1 + assert exits[1].exitcase == exits[1].llexitcase == linkcase2 + else: + assert exits[0].exitcase == exits[0].llexitcase == linkcase2 + assert exits[1].exitcase == exits[1].llexitcase == linkcase1 + def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: From arigo at codespeak.net Tue Apr 27 17:54:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 17:54:14 +0200 (CEST) Subject: [pypy-svn] r74112 - pypy/branch/blackhole-improvement/pypy/jit/codewriter/test Message-ID: <20100427155414.3C8B2282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 17:54:12 2010 New Revision: 74112 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Tweak the (skipped) test to match what I think should occur now. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 17:54:12 2010 @@ -1,4 +1,4 @@ -import py +import py, sys from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind @@ -335,14 +335,17 @@ return 42 except ZeroDivisionError: return -42 - + # 'int_add' and 'int_and' are used to detect the + # combination "%i0 = -sys.maxint-1, %i1 = -1". self.encoding_test(f, [7, 2], """ goto_if_not_int_is_true L1, %i1 - goto_if_div_overflow L2, %i0, %i1 - int_floordiv %i0, %i1, %i0 - int_return %i0 + int_add %i0, $MAXINT, %i2 + int_and %i2, %i1, %i3 + goto_if_not_int_ne L2, %i3, $-1 + int_floordiv %i0, %i1, %i4 + int_return %i4 L1: int_return $-42 L2: int_return $42 - """, transform=True) + """.replace('MAXINT', str(sys.maxint)), transform=True) From arigo at codespeak.net Tue Apr 27 18:09:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 18:09:21 +0200 (CEST) Subject: [pypy-svn] r74113 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427160921.33498282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 18:09:19 2010 New Revision: 74113 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Test and fix: we must rename variables on the links too. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Tue Apr 27 18:09:19 2010 @@ -25,27 +25,32 @@ def transform(self, graph): self.graph = graph for block in graph.iterblocks(): - if block.operations == (): - continue - rename = {} - newoperations = [] - if block.exitswitch == c_last_exception: - op_raising_exception = block.operations[-1] + self.optimize_block(block) + + def optimize_block(self, block): + if block.operations == (): + return + rename = {} + newoperations = [] + if block.exitswitch == c_last_exception: + op_raising_exception = block.operations[-1] + else: + op_raising_exception = Ellipsis + for op in block.operations: + try: + op1 = self.rewrite_operation(op) + except NoOp: + if op.result is not None: + rename[op.result] = rename.get(op.args[0], op.args[0]) + if op is op_raising_exception: + self.killed_exception_raising_operation(block) else: - op_raising_exception = Ellipsis - for op in block.operations: - try: - op1 = self.rewrite_operation(op) - except NoOp: - if op.result is not None: - rename[op.result] = rename.get(op.args[0], op.args[0]) - if op is op_raising_exception: - self.killed_exception_raising_operation(block) - else: - op2 = self.do_renaming(rename, op1) - newoperations.append(op2) - block.operations = newoperations - self.optimize_goto_if_not(block) + op2 = self.do_renaming(rename, op1) + newoperations.append(op2) + block.operations = newoperations + self.optimize_goto_if_not(block) + for link in block.exits: + self.do_renaming_on_link(rename, link) def do_renaming(self, rename, op): op = SpaceOperation(op.opname, op.args[:], op.result) @@ -62,6 +67,12 @@ op.args[i] = ListOfKind(v.kind, newlst) return op + def do_renaming_on_link(self, rename, link): + for i, v in enumerate(link.args): + if isinstance(v, Variable): + if v in rename: + link.args[i] = rename[v] + def killed_exception_raising_operation(self, block): assert block.exits[0].exitcase is None del block.exits[1:] Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Tue Apr 27 18:09:19 2010 @@ -160,3 +160,17 @@ fielddescr = ('fielddescr', S, name) assert op1.args == [v_parent, fielddescr] assert op1.result == v_result + +def test_rename_on_links(): + v1 = Variable() + v2 = Variable() + v3 = Variable() + block = Block([v1]) + block.operations = [SpaceOperation('cast_pointer', [v1], v2)] + block2 = Block([v3]) + block.closeblock(Link([v2], block2)) + Transformer().optimize_block(block) + assert block.inputargs == [v1] + assert block.operations == [] + assert block.exits[0].target is block2 + assert block.exits[0].args == [v1] From arigo at codespeak.net Tue Apr 27 18:13:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 18:13:03 +0200 (CEST) Subject: [pypy-svn] r74114 - pypy/branch/blackhole-improvement/pypy/translator/tool Message-ID: <20100427161303.D2AD0282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 18:13:02 2010 New Revision: 74114 Modified: pypy/branch/blackhole-improvement/pypy/translator/tool/make_dot.py Log: Fix to accept the strange format of flowgraphs produced by jit/codewriter/jitter.py. Modified: pypy/branch/blackhole-improvement/pypy/translator/tool/make_dot.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/translator/tool/make_dot.py (original) +++ pypy/branch/blackhole-improvement/pypy/translator/tool/make_dot.py Tue Apr 27 18:13:02 2010 @@ -152,7 +152,7 @@ shape = "octagon" if block.exitswitch is not None: - lines.append("exitswitch: %s" % block.exitswitch) + lines.append("exitswitch: %s" % (block.exitswitch,)) iargs = " ".join(map(repr, block.inputargs)) if self.VERBOSE: From fijal at codespeak.net Tue Apr 27 18:25:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 18:25:33 +0200 (CEST) Subject: [pypy-svn] r74115 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427162533.157ED282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 18:25:31 2010 New Revision: 74115 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Add a repr to register Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Tue Apr 27 18:25:31 2010 @@ -28,6 +28,8 @@ def __init__(self, kind, index): self.kind = kind # 'int', 'ref' or 'float' self.index = index + def __repr__(self): + return "%%%s%d" % (self.kind[0], self.index) class ListOfKind(object): # a list of Regs/Consts, all of the same 'kind'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 18:25:31 2010 @@ -1,7 +1,7 @@ import py, sys from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list -from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind +from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind, Register from pypy.jit.codewriter.format import format_assembler from pypy.jit.codewriter.jitter import transform_graph from pypy.jit.metainterp.history import AbstractDescr @@ -49,6 +49,8 @@ result = reorder_renaming_list([4, 3, 1, 2, 6], [1, 2, 3, 4, 5]) assert result == None +def test_repr(): + assert repr(Register('int', 13)) == '%i13' class TestFlatten: From exarkun at codespeak.net Tue Apr 27 18:29:07 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 27 Apr 2010 18:29:07 +0200 (CEST) Subject: [pypy-svn] r74116 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100427162907.528AB282B9D@codespeak.net> Author: exarkun Date: Tue Apr 27 18:29:05 2010 New Revision: 74116 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Add a test for an extension module with a dotted name Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Tue Apr 27 18:29:05 2010 @@ -132,7 +132,16 @@ cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") - def import_module(self, name, init=None, body='', load_it=True): + def import_module(self, name, init=None, body='', load_it=True, filename=None): + """ + init specifies the overall template of the module. + + if init is None, the module source will be loaded from a file in this + test direcory, give a name given by the filename parameter. + + if filename is None, the module name will be used to construct the + filename. + """ if init is not None: code = """ #include @@ -144,6 +153,8 @@ """ % dict(name=name, init=init, body=body) kwds = dict(separate_module_sources=[code]) else: + if filename is not None: + name = filename filename = py.path.local(autopath.pypydir) / 'module' \ / 'cpyext'/ 'test' / (name + ".c") kwds = dict(separate_module_files=[filename]) @@ -310,6 +321,18 @@ assert module.return_cookie() == 3.14 + def test_InitModule4Dotted(self): + """ + If the module name passed to Py_InitModule4 includes a package, only + the module name (the part after the last dot) is considered when + computing the name of the module initializer function. + """ + skip("This is not supported at present.") + expected_name = "pypy.module.cpyext.test.dotted" + module = self.import_module(name=expected_name, filename="dotted") + assert module.__name__ == expected_name + + def test_modinit_func(self): """ A module can use the PyMODINIT_FUNC macro to declare or define its From exarkun at codespeak.net Tue Apr 27 18:29:29 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 27 Apr 2010 18:29:29 +0200 (CEST) Subject: [pypy-svn] r74117 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100427162929.704F1282B9D@codespeak.net> Author: exarkun Date: Tue Apr 27 18:29:28 2010 New Revision: 74117 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/dotted.c Log: Add the necessary module source for test_InitModule4Dotted as well Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/dotted.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/dotted.c Tue Apr 27 18:29:28 2010 @@ -0,0 +1,10 @@ +#include "Python.h" + +static PyMethodDef dotted_functions[] = { + {NULL, NULL} +}; + +void initdotted(void) +{ + Py_InitModule("pypy.module.cpyext.test.dotted", dotted_functions); +} From exarkun at codespeak.net Tue Apr 27 18:39:44 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 27 Apr 2010 18:39:44 +0200 (CEST) Subject: [pypy-svn] r74118 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100427163944.86314282B9D@codespeak.net> Author: exarkun Date: Tue Apr 27 18:39:42 2010 New Revision: 74118 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Consider only the module name when computing the extension module initializer name; also fix a bug in import_module introduced in the previous change to that method Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Tue Apr 27 18:39:42 2010 @@ -738,7 +738,7 @@ "unable to load extension module '%s': %s", path, e.msg) try: - initptr = libffi.dlsym(dll.lib, 'init%s' % (name,)) + initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) except KeyError: raise operationerrfmt( space.w_ImportError, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Tue Apr 27 18:39:42 2010 @@ -153,10 +153,10 @@ """ % dict(name=name, init=init, body=body) kwds = dict(separate_module_sources=[code]) else: - if filename is not None: - name = filename + if filename is None: + filename = name filename = py.path.local(autopath.pypydir) / 'module' \ - / 'cpyext'/ 'test' / (name + ".c") + / 'cpyext'/ 'test' / (filename + ".c") kwds = dict(separate_module_files=[filename]) state = self.space.fromcache(State) @@ -327,7 +327,6 @@ the module name (the part after the last dot) is considered when computing the name of the module initializer function. """ - skip("This is not supported at present.") expected_name = "pypy.module.cpyext.test.dotted" module = self.import_module(name=expected_name, filename="dotted") assert module.__name__ == expected_name From arigo at codespeak.net Tue Apr 27 19:25:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 19:25:09 +0200 (CEST) Subject: [pypy-svn] r74119 - pypy/branch/blackhole-improvement/pypy/tool/algo Message-ID: <20100427172509.5A5DA282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 19:25:07 2010 New Revision: 74119 Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Log: Accept duplicate add_node()s. Modified: pypy/branch/blackhole-improvement/pypy/tool/algo/color.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/tool/algo/color.py (original) +++ pypy/branch/blackhole-improvement/pypy/tool/algo/color.py Tue Apr 27 19:25:07 2010 @@ -6,9 +6,9 @@ self.neighbours = {} def add_node(self, v): - assert v not in self.neighbours, "duplicate vertex %r" % (v,) - self._all_nodes.append(v) - self.neighbours[v] = set() + if v not in self.neighbours: + self._all_nodes.append(v) + self.neighbours[v] = set() def add_edge(self, v1, v2): assert v1 != v2 From arigo at codespeak.net Tue Apr 27 19:25:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 19:25:46 +0200 (CEST) Subject: [pypy-svn] r74120 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100427172546.8ED45282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 19:25:45 2010 New Revision: 74120 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Lots of messy but local code to transform graphs containing the operation int_floordiv_ovf_zer. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Tue Apr 27 19:25:45 2010 @@ -1,7 +1,8 @@ +import sys from pypy.rpython.lltypesystem import lltype, rstr from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation, Variable, Constant -from pypy.objspace.flow.model import c_last_exception +from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind @@ -24,7 +25,7 @@ def transform(self, graph): self.graph = graph - for block in graph.iterblocks(): + for block in list(graph.iterblocks()): self.optimize_block(block) def optimize_block(self, block): @@ -34,6 +35,7 @@ newoperations = [] if block.exitswitch == c_last_exception: op_raising_exception = block.operations[-1] + self.rewrite_exception_operation(op_raising_exception, block) else: op_raising_exception = Ellipsis for op in block.operations: @@ -86,7 +88,7 @@ if len(block.exits) != 2: return False v = block.exitswitch - if v.concretetype != lltype.Bool: + if isinstance(v, tuple) or v.concretetype != lltype.Bool: return False for link in block.exits: if v in link.args: @@ -120,6 +122,78 @@ # ---------- + def rewrite_exception_operation(self, op, block): + # mangling of the graph for exception-raising operations that are + # not simple calls + try: + rewrite = _rewrite_exc_ops[op.opname] + except KeyError: + pass + else: + rewrite(self, op, block) + + def rewrite_exc_op_int_floordiv_ovf_zer(self, op, block): + # See test_flatten.test_int_floordiv_ovf_zer for the mangling done here + usedvars = self.list_vars_in_use_at_end(block, op.args, op.result) + block.operations.pop() + [ovf_link, zer_link] = self.extract_exc_links(block, [ + OverflowError, ZeroDivisionError]) + block = self.write_new_condition(block, usedvars, zer_link, + ('int_is_true', op.args[1])) + v1 = Variable(); v1.concretetype = lltype.Signed + v2 = Variable(); v2.concretetype = lltype.Signed + block.operations += [ + SpaceOperation('int_add', [op.args[0], + Constant(sys.maxint, lltype.Signed)], + v1), + SpaceOperation('int_and', [v1, op.args[1]], v2), + ] + block = self.write_new_condition(block, usedvars, ovf_link, + ('int_ne', v2, Constant(-1, lltype.Signed))) + block.operations += [ + SpaceOperation('int_floordiv', op.args, op.result), + ] + + def list_vars_in_use_at_end(self, block, extravars=[], exclude=None): + lists = [extravars] + for link in block.exits: + lists.append(link.args) + in_use = set() + for lst in lists: + for v in lst: + if isinstance(v, Variable): + in_use.add(v) + if exclude in in_use: + in_use.remove(exclude) + return list(in_use) + + def extract_exc_links(self, block, exceptions): + # Remove the exception links from the block. The exception links must + # be catching exactly the listed 'exceptions'. We return them to the + # caller. + assert block.exits[0].exitcase is None + real_exc_links = dict([(link.exitcase, link) + for link in block.exits[1:]]) + assert dict.fromkeys(real_exc_links) == dict.fromkeys(exceptions) + block.recloseblock(block.exits[0]) + return [real_exc_links[exc] for exc in exceptions] + + def write_new_condition(self, block, usedvars, exc_link, exitswitch): + # Write an exit at the end of the block, going either to a new + # block that is the continuation of the current block, or (in case + # 'exitswitch' is False) following the given 'exc_link'. + newblock = Block(usedvars) + normal_link = Link(usedvars, newblock) + normal_link.exitcase = normal_link.llexitcase = True + exc_link.exitcase = exc_link.llexitcase = False + block.exitswitch = exitswitch + [exit] = block.exits + block.recloseblock(normal_link, exc_link) + newblock.closeblock(exit) + return newblock + + # ---------- + def rewrite_operation(self, op): try: rewrite = _rewrite_ops[op.opname] @@ -265,10 +339,15 @@ # ____________________________________________________________ -_rewrite_ops = {} -for _name in dir(Transformer): - if _name.startswith('rewrite_op_'): - _rewrite_ops[_name[len('rewrite_op_'):]] = getattr(Transformer, _name) +def _with_prefix(prefix): + result = {} + for name in dir(Transformer): + if name.startswith(prefix): + result[name[len(prefix):]] = getattr(Transformer, name) + return result + +_rewrite_ops = _with_prefix('rewrite_op_') +_rewrite_exc_ops = _with_prefix('rewrite_exc_op_') primitive_type_size = { lltype.Signed: 'i', Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Tue Apr 27 19:25:45 2010 @@ -329,8 +329,9 @@ """, transform=True) def test_int_floordiv_ovf_zer(self): - py.test.skip("in-progress") def f(i, j): + assert i >= 0 + assert j >= 0 try: return ovfcheck(i // j) except OverflowError: @@ -346,8 +347,8 @@ goto_if_not_int_ne L2, %i3, $-1 int_floordiv %i0, %i1, %i4 int_return %i4 - L1: - int_return $-42 L2: int_return $42 + L1: + int_return $-42 """.replace('MAXINT', str(sys.maxint)), transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Tue Apr 27 19:25:45 2010 @@ -1,12 +1,14 @@ -import py +import py, sys from pypy.jit.codewriter import support from pypy.jit.codewriter.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.jitter import transform_graph from pypy.objspace.flow.model import Variable, Constant, SpaceOperation from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import c_last_exception from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rlib.rarithmetic import ovfcheck class TestRegAlloc: @@ -15,10 +17,12 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def check_assembler(self, graph, expected): - """Only for simple graphs. More complex graphs must first be - transformed by jitter.py before they can be subjected to - register allocation and flattening.""" + def check_assembler(self, graph, expected, transform=False): + # 'transform' can be False only for simple graphs. More complex + # graphs must first be transformed by jitter.py before they can be + # subjected to register allocation and flattening. + if transform: + transform_graph(graph) regalloc = perform_register_allocation(graph, 'int') ssarepr = flatten_graph(graph, {'int': regalloc}) asm = format_assembler(ssarepr) @@ -176,3 +180,27 @@ L2: reraise """) + + def test_int_floordiv_ovf_zer(self): + def f(i, j): + assert i >= 0 + assert j >= 0 + try: + return ovfcheck(i // j) + except OverflowError: + return 42 + except ZeroDivisionError: + return -42 + graph = self.make_graphs(f, [5, 6])[0] + self.check_assembler(graph, """ + goto_if_not_int_is_true L1, %i1 + int_add %i0, $MAXINT, %i2 + int_and %i2, %i1, %i2 + goto_if_not_int_ne L2, %i2, $-1 + int_floordiv %i0, %i1, %i0 + int_return %i0 + L2: + int_return $42 + L1: + int_return $-42 + """.replace('MAXINT', str(sys.maxint)), transform=True) From arigo at codespeak.net Tue Apr 27 19:44:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Apr 2010 19:44:54 +0200 (CEST) Subject: [pypy-svn] r74121 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100427174454.8FF92282B9D@codespeak.net> Author: arigo Date: Tue Apr 27 19:44:53 2010 New Revision: 74121 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Fixes, just enough for the test in test_basic. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Tue Apr 27 19:44:53 2010 @@ -88,7 +88,8 @@ if len(block.exits) != 2: return False v = block.exitswitch - if isinstance(v, tuple) or v.concretetype != lltype.Bool: + if (v == c_last_exception or isinstance(v, tuple) + or v.concretetype != lltype.Bool): return False for link in block.exits: if v in link.args: @@ -132,7 +133,7 @@ else: rewrite(self, op, block) - def rewrite_exc_op_int_floordiv_ovf_zer(self, op, block): + def _rewrite_exc_div_or_mod_ovf_zer(self, op, block): # See test_flatten.test_int_floordiv_ovf_zer for the mangling done here usedvars = self.list_vars_in_use_at_end(block, op.args, op.result) block.operations.pop() @@ -150,10 +151,14 @@ ] block = self.write_new_condition(block, usedvars, ovf_link, ('int_ne', v2, Constant(-1, lltype.Signed))) + basename = op.opname.split('_')[1] block.operations += [ - SpaceOperation('int_floordiv', op.args, op.result), + SpaceOperation('int_' + basename, op.args, op.result), ] + rewrite_exc_op_int_floordiv_ovf_zer = _rewrite_exc_div_or_mod_ovf_zer + rewrite_exc_op_int_mod_ovf_zer = _rewrite_exc_div_or_mod_ovf_zer + def list_vars_in_use_at_end(self, block, extravars=[], exclude=None): lists = [extravars] for link in block.exits: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Tue Apr 27 19:44:53 2010 @@ -277,6 +277,18 @@ return a * b @arguments("i", "i", returns="i") + def opimpl_int_and(self, a, b): + return a & b + + @arguments("i", "i", returns="i") + def opimpl_int_floordiv(self, a, b): + return llop.int_floordiv(lltype.Signed, a, b) + + @arguments("i", "i", returns="i") + def opimpl_int_mod(self, a, b): + return llop.int_mod(lltype.Signed, a, b) + + @arguments("i", "i", returns="i") def opimpl_uint_floordiv(self, a, b): c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) return intmask(c) @@ -344,6 +356,13 @@ else: return target + @arguments("L", "i", "pc", returns="L") + def opimpl_goto_if_not_int_is_true(self, target, a, pc): + if a: + return pc + else: + return target + @arguments("L", returns="L") def opimpl_goto(self, target): return target Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Tue Apr 27 19:44:53 2010 @@ -761,21 +761,26 @@ def test_zerodivisionerror(self): # test the case of exception-raising operation that is not delegated # to the backend at all: ZeroDivisionError - from pypy.rpython.lltypesystem.lloperation import llop # def f(n): + assert n >= 0 try: - return llop.int_mod_ovf_zer(lltype.Signed, 5, n) + return ovfcheck(5 % n) except ZeroDivisionError: return -666 + except OverflowError: + return -777 res = self.interp_operations(f, [0]) assert res == -666 # def f(n): + assert n >= 0 try: - return llop.int_floordiv_ovf_zer(lltype.Signed, 6, n) + return ovfcheck(6 // n) except ZeroDivisionError: return -667 + except OverflowError: + return -778 res = self.interp_operations(f, [0]) assert res == -667 From benjamin at codespeak.net Tue Apr 27 22:29:24 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 27 Apr 2010 22:29:24 +0200 (CEST) Subject: [pypy-svn] r74124 - pypy/trunk/pypy/rpython Message-ID: <20100427202924.9FA3F282B9D@codespeak.net> Author: benjamin Date: Tue Apr 27 22:29:22 2010 New Revision: 74124 Modified: pypy/trunk/pypy/rpython/rtyper.py Log: compare classes with 'is' not == Modified: pypy/trunk/pypy/rpython/rtyper.py ============================================================================== --- pypy/trunk/pypy/rpython/rtyper.py (original) +++ pypy/trunk/pypy/rpython/rtyper.py Tue Apr 27 22:29:22 2010 @@ -171,7 +171,7 @@ def getrepr(self, s_obj): # s_objs are not hashable... try hard to find a unique key anyway key = self.makekey(s_obj) - assert key[0] == s_obj.__class__ + assert key[0] is s_obj.__class__ try: result = self.reprs[key] except KeyError: From fijal at codespeak.net Tue Apr 27 22:32:12 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Apr 2010 22:32:12 +0200 (CEST) Subject: [pypy-svn] r74125 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100427203212.907F7282B9D@codespeak.net> Author: fijal Date: Tue Apr 27 22:32:10 2010 New Revision: 74125 Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pythread.h Log: Empty file, needed by some extensions. Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/pythread.h ============================================================================== From arigo at codespeak.net Wed Apr 28 10:39:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 10:39:33 +0200 (CEST) Subject: [pypy-svn] r74152 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428083933.9FCB2282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 10:39:31 2010 New Revision: 74152 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Log: Give up on the complicated graph mangling code in jitter.py and instead just replace 'int_{floordiv,mod}_ovf_zer' with a call to a helper function. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Wed Apr 28 10:39:31 2010 @@ -131,9 +131,14 @@ def make_exception_link(self, link): # Like make_link(), but also introduces the 'last_exception' and - # 'last_exc_value' as variables if needed + # 'last_exc_value' as variables if needed. Also check if the link + # is jumping directly to the re-raising exception block. assert link.last_exception is not None assert link.last_exc_value is not None + if link.target.operations == () and link.args == [link.last_exception, + link.last_exc_value]: + self.emitline("reraise") + return # done if link.last_exception in link.args: self.emitline("last_exception", self.getcolor(link.last_exception)) if link.last_exc_value in link.args: @@ -157,13 +162,7 @@ for link in block.exits[1:]: if link.exitcase is Exception: # this link captures all exceptions - if (link.target.operations == () - and link.args == [link.last_exception, - link.last_exc_value]): - # the link is going directly to the except block - self.emitline("reraise") - else: - self.make_exception_link(link) + self.make_exception_link(link) break self.emitline('goto_if_exception_mismatch', Constant(link.llexitcase, Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 10:39:31 2010 @@ -35,7 +35,6 @@ newoperations = [] if block.exitswitch == c_last_exception: op_raising_exception = block.operations[-1] - self.rewrite_exception_operation(op_raising_exception, block) else: op_raising_exception = Ellipsis for op in block.operations: @@ -123,82 +122,6 @@ # ---------- - def rewrite_exception_operation(self, op, block): - # mangling of the graph for exception-raising operations that are - # not simple calls - try: - rewrite = _rewrite_exc_ops[op.opname] - except KeyError: - pass - else: - rewrite(self, op, block) - - def _rewrite_exc_div_or_mod_ovf_zer(self, op, block): - # See test_flatten.test_int_floordiv_ovf_zer for the mangling done here - usedvars = self.list_vars_in_use_at_end(block, op.args, op.result) - block.operations.pop() - [ovf_link, zer_link] = self.extract_exc_links(block, [ - OverflowError, ZeroDivisionError]) - block = self.write_new_condition(block, usedvars, zer_link, - ('int_is_true', op.args[1])) - v1 = Variable(); v1.concretetype = lltype.Signed - v2 = Variable(); v2.concretetype = lltype.Signed - block.operations += [ - SpaceOperation('int_add', [op.args[0], - Constant(sys.maxint, lltype.Signed)], - v1), - SpaceOperation('int_and', [v1, op.args[1]], v2), - ] - block = self.write_new_condition(block, usedvars, ovf_link, - ('int_ne', v2, Constant(-1, lltype.Signed))) - basename = op.opname.split('_')[1] - block.operations += [ - SpaceOperation('int_' + basename, op.args, op.result), - ] - - rewrite_exc_op_int_floordiv_ovf_zer = _rewrite_exc_div_or_mod_ovf_zer - rewrite_exc_op_int_mod_ovf_zer = _rewrite_exc_div_or_mod_ovf_zer - - def list_vars_in_use_at_end(self, block, extravars=[], exclude=None): - lists = [extravars] - for link in block.exits: - lists.append(link.args) - in_use = set() - for lst in lists: - for v in lst: - if isinstance(v, Variable): - in_use.add(v) - if exclude in in_use: - in_use.remove(exclude) - return list(in_use) - - def extract_exc_links(self, block, exceptions): - # Remove the exception links from the block. The exception links must - # be catching exactly the listed 'exceptions'. We return them to the - # caller. - assert block.exits[0].exitcase is None - real_exc_links = dict([(link.exitcase, link) - for link in block.exits[1:]]) - assert dict.fromkeys(real_exc_links) == dict.fromkeys(exceptions) - block.recloseblock(block.exits[0]) - return [real_exc_links[exc] for exc in exceptions] - - def write_new_condition(self, block, usedvars, exc_link, exitswitch): - # Write an exit at the end of the block, going either to a new - # block that is the continuation of the current block, or (in case - # 'exitswitch' is False) following the given 'exc_link'. - newblock = Block(usedvars) - normal_link = Link(usedvars, newblock) - normal_link.exitcase = normal_link.llexitcase = True - exc_link.exitcase = exc_link.llexitcase = False - block.exitswitch = exitswitch - [exit] = block.exits - block.recloseblock(normal_link, exc_link) - newblock.closeblock(exit) - return newblock - - # ---------- - def rewrite_operation(self, op): try: rewrite = _rewrite_ops[op.opname] @@ -247,6 +170,21 @@ else: raise AssertionError(kind) lst.append(v) + def _rewrite_with_helper(self, op): + from pypy.jit.codewriter.support import builtin_func_for_spec + c_func, _ = builtin_func_for_spec(self.cpu.rtyper, op.opname, + [lltype.Signed, lltype.Signed], + lltype.Signed) + op = SpaceOperation('direct_call', [c_func] + op.args, op.result) + return self.rewrite_op_direct_call(op) + + rewrite_op_int_floordiv_ovf_zer = _rewrite_with_helper + rewrite_op_int_floordiv_ovf = _rewrite_with_helper + rewrite_op_int_floordiv_zer = _rewrite_with_helper + rewrite_op_int_mod_ovf_zer = _rewrite_with_helper + rewrite_op_int_mod_ovf = _rewrite_with_helper + rewrite_op_int_mod_zer = _rewrite_with_helper + def rewrite_op_hint(self, op): hints = op.args[1].value if hints.get('promote') and op.args[0].concretetype is not lltype.Void: @@ -351,8 +289,7 @@ result[name[len(prefix):]] = getattr(Transformer, name) return result -_rewrite_ops = _with_prefix('rewrite_op_') -_rewrite_exc_ops = _with_prefix('rewrite_exc_op_') +_rewrite_ops = _with_prefix('rewrite_op_') primitive_type_size = { lltype.Signed: 'i', Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed Apr 28 10:39:31 2010 @@ -143,6 +143,20 @@ def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) +def _ll_2_int_floordiv_ovf_zer(x, y): + return llop.int_floordiv_ovf_zer(lltype.Signed, x, y) +def _ll_2_int_floordiv_ovf(x, y): + return llop.int_floordiv_ovf(lltype.Signed, x, y) +def _ll_2_int_floordiv_zer(x, y): + return llop.int_floordiv_zer(lltype.Signed, x, y) + +def _ll_2_int_mod_ovf_zer(x, y): + return llop.int_mod_ovf_zer(lltype.Signed, x, y) +def _ll_2_int_mod_ovf(x, y): + return llop.int_mod_ovf(lltype.Signed, x, y) +def _ll_2_int_mod_zer(x, y): + return llop.int_mod_zer(lltype.Signed, x, y) + class LLtypeHelpers: Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Wed Apr 28 10:39:31 2010 @@ -26,7 +26,18 @@ def __repr__(self): return '' +class FakeDict(object): + def __getitem__(self, key): + F = lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + f = lltype.functionptr(F, key[0]) + c_func = Constant(f, lltype.typeOf(f)) + return c_func, lltype.Signed + +class FakeRTyper(object): + _builtin_func_for_spec_cache = FakeDict() + class FakeCPU: + rtyper = FakeRTyper() def calldescrof(self, FUNC, ARGS, RESULT): return FakeDescr() def fielddescrof(self, STRUCT, name): @@ -338,17 +349,53 @@ return 42 except ZeroDivisionError: return -42 - # 'int_add' and 'int_and' are used to detect the - # combination "%i0 = -sys.maxint-1, %i1 = -1". self.encoding_test(f, [7, 2], """ - goto_if_not_int_is_true L1, %i1 - int_add %i0, $MAXINT, %i2 - int_and %i2, %i1, %i3 - goto_if_not_int_ne L2, %i3, $-1 - int_floordiv %i0, %i1, %i4 - int_return %i4 + residual_call_ir_i $<* fn int_floordiv_ovf_zer>, , I[%i0, %i1], R[], %i2 + catch_exception L1 + int_return %i2 + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 + int_return $42 L2: + goto_if_exception_mismatch $<* struct object_vtable>, L3 + int_return $-42 + L3: + reraise + """, transform=True) + + def test_int_mod_ovf(self): + def f(i, j): + assert i >= 0 + assert j >= 0 + try: + return ovfcheck(i % j) + except OverflowError: + return 42 + # XXX so far, this really produces a int_mod_ovf_zer... + self.encoding_test(f, [7, 2], """ + residual_call_ir_i $<* fn int_mod_ovf_zer>, , I[%i0, %i1], R[], %i2 + catch_exception L1 + int_return %i2 + L1: + goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $42 + L2: + reraise + """, transform=True) + + def test_int_add_ovf(self): + def f(i, j): + try: + return ovfcheck(i + j) + except OverflowError: + return 42 + self.encoding_test(f, [7, 2], """ + int_add_ovf %i0, %i1, %i2 + catch_exception L1 + int_return %i2 L1: - int_return $-42 - """.replace('MAXINT', str(sys.maxint)), transform=True) + goto_if_exception_mismatch $<* struct object_vtable>, L2 + int_return $42 + L2: + reraise + """, transform=True) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Wed Apr 28 10:39:31 2010 @@ -180,27 +180,3 @@ L2: reraise """) - - def test_int_floordiv_ovf_zer(self): - def f(i, j): - assert i >= 0 - assert j >= 0 - try: - return ovfcheck(i // j) - except OverflowError: - return 42 - except ZeroDivisionError: - return -42 - graph = self.make_graphs(f, [5, 6])[0] - self.check_assembler(graph, """ - goto_if_not_int_is_true L1, %i1 - int_add %i0, $MAXINT, %i2 - int_and %i2, %i1, %i2 - goto_if_not_int_ne L2, %i2, $-1 - int_floordiv %i0, %i1, %i0 - int_return %i0 - L2: - int_return $42 - L1: - int_return $-42 - """.replace('MAXINT', str(sys.maxint)), transform=True) From arigo at codespeak.net Wed Apr 28 11:39:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 11:39:47 +0200 (CEST) Subject: [pypy-svn] r74153 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428093947.ED95B282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 11:39:46 2010 New Revision: 74153 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/heaptracker.py - copied unchanged from r74151, pypy/branch/blackhole-improvement/pypy/jit/metainterp/heaptracker.py Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Mallocs. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 11:39:46 2010 @@ -4,6 +4,7 @@ from pypy.objspace.flow.model import SpaceOperation, Variable, Constant from pypy.objspace.flow.model import Block, Link, c_last_exception from pypy.jit.codewriter.flatten import ListOfKind +from pypy.jit.codewriter import support, heaptracker def transform_graph(graph, cpu=None): @@ -170,20 +171,23 @@ else: raise AssertionError(kind) lst.append(v) - def _rewrite_with_helper(self, op): - from pypy.jit.codewriter.support import builtin_func_for_spec - c_func, _ = builtin_func_for_spec(self.cpu.rtyper, op.opname, - [lltype.Signed, lltype.Signed], - lltype.Signed) - op = SpaceOperation('direct_call', [c_func] + op.args, op.result) - return self.rewrite_op_direct_call(op) - - rewrite_op_int_floordiv_ovf_zer = _rewrite_with_helper - rewrite_op_int_floordiv_ovf = _rewrite_with_helper - rewrite_op_int_floordiv_zer = _rewrite_with_helper - rewrite_op_int_mod_ovf_zer = _rewrite_with_helper - rewrite_op_int_mod_ovf = _rewrite_with_helper - rewrite_op_int_mod_zer = _rewrite_with_helper + def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None): + if oopspec_name is None: oopspec_name = op.opname + if args is None: args = op.args + argtypes = [v.concretetype for v in args] + resulttype = op.result.concretetype + c_func, TP = support.builtin_func_for_spec(self.cpu.rtyper, + oopspec_name, argtypes, + resulttype, extra) + op1 = SpaceOperation('direct_call', [c_func] + args, op.result) + return self.rewrite_op_direct_call(op1) + + rewrite_op_int_floordiv_ovf_zer = _do_builtin_call + rewrite_op_int_floordiv_ovf = _do_builtin_call + rewrite_op_int_floordiv_zer = _do_builtin_call + rewrite_op_int_mod_ovf_zer = _do_builtin_call + rewrite_op_int_mod_ovf = _do_builtin_call + rewrite_op_int_mod_zer = _do_builtin_call def rewrite_op_hint(self, op): hints = op.args[1].value @@ -280,6 +284,32 @@ return SpaceOperation('getfield_%s_%s%s' % (argname, kind, pure), [v_inst, descr], op.result) + def rewrite_op_malloc(self, op): + assert op.args[1].value == {'flavor': 'gc'} + STRUCT = op.args[0].value + vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT) + if vtable: + # do we have a __del__? + try: + rtti = lltype.getRuntimeTypeInfo(STRUCT) + except ValueError: + pass + else: + if hasattr(rtti._obj, 'destructor_funcptr'): + RESULT = lltype.Ptr(STRUCT) + assert RESULT == op.result.concretetype + return self._do_builtin_call(op, 'alloc_with_del', + [], extra = (RESULT, vtable)) + # store the vtable as an address -- that's fine, because the + # GC doesn't need to follow them + #self.codewriter.register_known_gctype(vtable, STRUCT) + sizevtabledescr = self.cpu.sizevtableof(STRUCT, vtable) + return SpaceOperation('new_with_vtable', [sizevtabledescr], + op.result) + else: + sizedescr = self.cpu.sizeof(STRUCT) + return SpaceOperation('new', [sizedescr], op.result) + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/support.py Wed Apr 28 11:39:46 2010 @@ -210,11 +210,12 @@ # ---------- malloc with del ---------- - def _ll_1_alloc_with_del(RESULT, vtable): - p = lltype.malloc(RESULT) - lltype.cast_pointer(rclass.OBJECTPTR, p).typeptr = vtable - return p - _ll_1_alloc_with_del.need_result_type = True + def build_ll_0_alloc_with_del(RESULT, vtable): + def _ll_0_alloc_with_del(): + p = lltype.malloc(RESULT) + lltype.cast_pointer(rclass.OBJECTPTR, p).typeptr = vtable + return p + return _ll_0_alloc_with_del class OOtypeHelpers: @@ -294,8 +295,10 @@ # ------------------------------------------------------- -def setup_extra_builtin(rtyper, oopspec_name, nb_args): +def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None): name = '_ll_%d_%s' % (nb_args, oopspec_name.replace('.', '_')) + if extra is not None: + name = 'build' + name try: wrapper = globals()[name] except KeyError: @@ -304,6 +307,8 @@ else: Helpers = OOtypeHelpers wrapper = getattr(Helpers, name).im_func + if extra is not None: + wrapper = wrapper(*extra) return wrapper # # ____________________________________________________________ @@ -388,8 +393,8 @@ else: raise ValueError(op.opname) -def builtin_func_for_spec(rtyper, oopspec_name, ll_args, ll_res): - key = (oopspec_name, tuple(ll_args), ll_res) +def builtin_func_for_spec(rtyper, oopspec_name, ll_args, ll_res, extra=None): + key = (oopspec_name, tuple(ll_args), ll_res, extra) try: return rtyper._builtin_func_for_spec_cache[key] except (KeyError, AttributeError): @@ -400,14 +405,19 @@ else: LIST_OR_DICT = ll_args[0] s_result = annmodel.lltype_to_annotation(ll_res) - impl = setup_extra_builtin(rtyper, oopspec_name, len(args_s)) + impl = setup_extra_builtin(rtyper, oopspec_name, len(args_s), extra) if getattr(impl, 'need_result_type', False): bk = rtyper.annotator.bookkeeper args_s.insert(0, annmodel.SomePBC([bk.getdesc(deref(ll_res))])) # - mixlevelann = MixLevelHelperAnnotator(rtyper) - c_func = mixlevelann.constfunc(impl, args_s, s_result) - mixlevelann.finish() + if hasattr(rtyper, 'annotator'): # regular case + mixlevelann = MixLevelHelperAnnotator(rtyper) + c_func = mixlevelann.constfunc(impl, args_s, s_result) + mixlevelann.finish() + else: + # for testing only + c_func = Constant(oopspec_name, + lltype.Ptr(lltype.FuncType(ll_args, ll_res))) # if not hasattr(rtyper, '_builtin_func_for_spec_cache'): rtyper._builtin_func_for_spec_cache = {} Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 11:39:46 2010 @@ -5,12 +5,22 @@ from pypy.jit.metainterp.history import getkind from pypy.rpython.lltypesystem import lltype, rclass, rstr from pypy.translator.unsimplify import varoftype +from pypy.jit.codewriter import heaptracker + +class FakeRTyper: + class type_system: name = 'lltypesystem' + instance_reprs = {} class FakeCPU: + rtyper = FakeRTyper() def calldescrof(self, FUNC, ARGS, RESULT): return ('calldescr', FUNC, ARGS, RESULT) def fielddescrof(self, STRUCT, name): return ('fielddescr', STRUCT, name) + def sizeof(self, STRUCT): + return ('sizedescr', STRUCT) + def sizevtableof(self, STRUCT, vtable): + return ('sizevtabledescr', STRUCT, vtable) class FakeLink: args = [] @@ -161,6 +171,41 @@ assert op1.args == [v_parent, fielddescr] assert op1.result == v_result +def test_malloc_new(): + S = lltype.GcStruct('S') + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc'}, lltype.Void)], v) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new' + assert op1.args == [('sizedescr', S)] + +def test_malloc_new_with_vtable(): + class vtable: pass + S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) + heaptracker.set_testing_vtable_for_gcstruct(S, vtable, 'S') + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc'}, lltype.Void)], v) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'new_with_vtable' + assert op1.args == [('sizevtabledescr', S, vtable)] + +def test_malloc_new_with_destructor(): + class vtable: pass + S = lltype.GcStruct('S', ('parent', rclass.OBJECT)) + DESTRUCTOR = lltype.FuncType([lltype.Ptr(S)], lltype.Void) + destructor = lltype.functionptr(DESTRUCTOR, 'destructor') + lltype.attachRuntimeTypeInfo(S, destrptr=destructor) + heaptracker.set_testing_vtable_for_gcstruct(S, vtable, 'S') + v = varoftype(lltype.Ptr(S)) + op = SpaceOperation('malloc', [Constant(S, lltype.Void), + Constant({'flavor': 'gc'}, lltype.Void)], v) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'residual_call_r_r' + assert op1.args[0].value == 'alloc_with_del' # pseudo-function as a str + assert list(op1.args[2]) == [] + def test_rename_on_links(): v1 = Variable() v2 = Variable() From arigo at codespeak.net Wed Apr 28 11:54:50 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 11:54:50 +0200 (CEST) Subject: [pypy-svn] r74154 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter metainterp Message-ID: <20100428095450.F34C0282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 11:54:48 2010 New Revision: 74154 Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/heaptracker.py Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Enough to pass test_isinstance. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/llimpl.py Wed Apr 28 11:54:48 2010 @@ -1151,16 +1151,6 @@ def get_frame_forced_token(opaque_frame): return llmemory.cast_ptr_to_adr(opaque_frame) -class MemoCast(object): - def __init__(self): - self.addresses = [llmemory.NULL] - self.rev_cache = {} - self.vtable_to_size = {} - -def new_memo_cast(): - memocast = MemoCast() - return _to_opaque(memocast) - ##def cast_adr_to_int(memocast, adr): ## # xxx slow ## assert lltype.typeOf(adr) == llmemory.Address @@ -1280,11 +1270,11 @@ newvalue = cast_from_ptr(ITEMTYPE, newvalue) array.setitem(index, newvalue) -def do_setfield_gc_int(struct, fieldnum, newvalue, memocast): +def do_setfield_gc_int(struct, fieldnum, newvalue): STRUCT, fieldname = symbolic.TokenToField[fieldnum] ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_int(FIELDTYPE, newvalue, memocast) + newvalue = cast_from_int(FIELDTYPE, newvalue) setattr(ptr, fieldname, newvalue) def do_setfield_gc_float(struct, fieldnum, newvalue): @@ -1480,16 +1470,13 @@ COMPILEDLOOP = lltype.Ptr(lltype.OpaqueType("CompiledLoop")) FRAME = lltype.Ptr(lltype.OpaqueType("Frame")) OOFRAME = lltype.Ptr(lltype.OpaqueType("OOFrame")) -MEMOCAST = lltype.Ptr(lltype.OpaqueType("MemoCast")) _TO_OPAQUE[CompiledLoop] = COMPILEDLOOP.TO _TO_OPAQUE[Frame] = FRAME.TO _TO_OPAQUE[OOFrame] = OOFRAME.TO -_TO_OPAQUE[MemoCast] = MEMOCAST.TO s_CompiledLoop = annmodel.SomePtr(COMPILEDLOOP) s_Frame = annmodel.SomePtr(FRAME) -s_MemoCast = annmodel.SomePtr(MEMOCAST) setannotation(compile_start, s_CompiledLoop) setannotation(compile_start_int_var, annmodel.SomeInteger()) @@ -1530,11 +1517,6 @@ setannotation(get_forced_token_frame, s_Frame) setannotation(get_frame_forced_token, annmodel.SomeAddress()) -setannotation(new_memo_cast, s_MemoCast) -##setannotation(cast_adr_to_int, annmodel.SomeInteger()) -##setannotation(cast_int_to_adr, annmodel.SomeAddress()) -setannotation(set_class_size, annmodel.s_None) - setannotation(do_arraylen_gc, annmodel.SomeInteger()) setannotation(do_strlen, annmodel.SomeInteger()) setannotation(do_strgetitem, annmodel.SomeInteger()) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed Apr 28 11:54:48 2010 @@ -86,11 +86,11 @@ self.stats.exec_counters = {} self.stats.exec_jumps = 0 self.stats.exec_conditional_jumps = 0 - self.memo_cast = llimpl.new_memo_cast() llimpl._stats = self.stats llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] self._descrs = {} + self.class_vtables_as_int = {} def _freeze_(self): assert self.translate_support_code @@ -105,13 +105,6 @@ self._descrs[key] = descr return descr - def set_class_sizes(self, class_sizes): - self.class_sizes = class_sizes - for vtable, size in class_sizes.items(): - if not self.is_oo: - size = size.ofs - llimpl.set_class_size(self.memo_cast, vtable, size) - def compile_bridge(self, faildescr, inputargs, operations): c = llimpl.compile_start() self._compile_loop_or_bridge(c, inputargs, operations) @@ -285,6 +278,14 @@ assert not isinstance(S, lltype.Ptr) return self.getdescr(symbolic.get_size(S)) + def sizevtableof(self, S, vtable): + assert isinstance(S, lltype.GcStruct) + descr = self.getdescr(symbolic.get_size(S)) + vtable_adr = llmemory.cast_ptr_to_adr(vtable) + vtable_int = llmemory.cast_adr_to_int(vtable_adr) + self.class_vtables_as_int[descr] = vtable_int + return descr + def cast_adr_to_int(self, adr): return llimpl.cast_adr_to_int(self.memo_cast, adr) @@ -427,13 +428,11 @@ assert isinstance(size, Descr) return history.BoxPtr(llimpl.do_new(size.ofs)) - def do_new_with_vtable(self, vtablebox): - vtable = vtablebox.getint() - size = self.class_sizes[vtable] - result = llimpl.do_new(size.ofs) + def bh_new_with_vtable(self, sizevtabledescr): + result = llimpl.do_new(sizevtabledescr.ofs) llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, - vtable, self.memo_cast) - return history.BoxPtr(result) + self.class_vtables_as_int[sizevtabledescr]) + return result def do_new_array(self, countbox, size): assert isinstance(size, Descr) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed Apr 28 11:54:48 2010 @@ -24,9 +24,6 @@ def get_fail_descr_from_number(self, n): return self.fail_descr_list[n] - def set_class_sizes(self, class_sizes): - self.class_sizes = class_sizes - def setup_once(self): """Called once by the front-end when the program starts.""" pass @@ -118,7 +115,7 @@ raise NotImplementedError @staticmethod - def numof(S): + def sizevtableof(S, vtable): raise NotImplementedError @staticmethod @@ -215,6 +212,9 @@ def do_new_with_vtable(self, classbox): raise NotImplementedError + def bh_new_with_vtable(self, sizevtabledescr): + raise NotImplementedError + def do_new_array(self, lengthbox, arraydescr): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 11:54:48 2010 @@ -239,9 +239,8 @@ return ARRAY.OF == lltype.Void def rewrite_op_getfield(self, op): - #if self.is_typeptr_getset(op): - # self.handle_getfield_typeptr(op) - # return + if self.is_typeptr_getset(op): + XXX # turn the flow graph 'getfield' operation into our own version [v_inst, c_fieldname] = op.args RESULT = op.result.concretetype @@ -284,6 +283,16 @@ return SpaceOperation('getfield_%s_%s%s' % (argname, kind, pure), [v_inst, descr], op.result) + def rewrite_op_setfield(self, op): + if self.is_typeptr_getset(op): + # ignore the operation completely -- instead, it's done by 'new' + raise NoOp + XXX + + def is_typeptr_getset(self, op): + return (op.args[1].value == 'typeptr' and + op.args[0].concretetype.TO._hints.get('typeptr')) + def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} STRUCT = op.args[0].value Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 11:54:48 2010 @@ -504,3 +504,7 @@ opimpl_getfield_raw_u_pure = opimpl_getfield_raw_u opimpl_getfield_raw_r_pure = opimpl_getfield_raw_r opimpl_getfield_raw_f_pure = opimpl_getfield_raw_f + + @arguments("d", returns="r") + def opimpl_new_with_vtable(self, descr): + return self.cpu.bh_new_with_vtable(descr) From arigo at codespeak.net Wed Apr 28 13:04:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:04:39 +0200 (CEST) Subject: [pypy-svn] r74155 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter codewriter/test metainterp Message-ID: <20100428110439.0E626282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:04:38 2010 New Revision: 74155 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: getfield('typeptr'). Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed Apr 28 13:04:38 2010 @@ -434,6 +434,12 @@ self.class_vtables_as_int[sizevtabledescr]) return result + def bh_classof(self, struct): + struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) + result = struct.typeptr + result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) + return llmemory.cast_adr_to_int(result_adr) + def do_new_array(self, countbox, size): assert isinstance(size, Descr) count = countbox.getint() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed Apr 28 13:04:38 2010 @@ -215,6 +215,9 @@ def bh_new_with_vtable(self, sizevtabledescr): raise NotImplementedError + def bh_classof(self, struct): + raise NotImplementedError + def do_new_array(self, lengthbox, arraydescr): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:04:38 2010 @@ -240,7 +240,7 @@ def rewrite_op_getfield(self, op): if self.is_typeptr_getset(op): - XXX + return self.handle_getfield_typeptr(op) # turn the flow graph 'getfield' operation into our own version [v_inst, c_fieldname] = op.args RESULT = op.result.concretetype @@ -293,6 +293,9 @@ return (op.args[1].value == 'typeptr' and op.args[0].concretetype.TO._hints.get('typeptr')) + def handle_getfield_typeptr(self, op): + return SpaceOperation('classof', [op.args[0]], op.result) + def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} STRUCT = op.args[0].value Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 13:04:38 2010 @@ -171,6 +171,16 @@ assert op1.args == [v_parent, fielddescr] assert op1.result == v_result +def test_getfield_typeptr(): + v_parent = varoftype(rclass.OBJECTPTR) + c_name = Constant('typeptr', lltype.Void) + v_result = varoftype(rclass.OBJECT.typeptr) + op = SpaceOperation('getfield', [v_parent, c_name], v_result) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'classof' + assert op1.args == [v_parent] + assert op1.result == v_result + def test_malloc_new(): S = lltype.GcStruct('S') v = varoftype(lltype.Ptr(S)) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 13:04:38 2010 @@ -508,3 +508,7 @@ @arguments("d", returns="r") def opimpl_new_with_vtable(self, descr): return self.cpu.bh_new_with_vtable(descr) + + @arguments("r", returns="i") + def opimpl_classof(self, struct): + return self.cpu.bh_classof(struct) From arigo at codespeak.net Wed Apr 28 13:13:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:13:41 +0200 (CEST) Subject: [pypy-svn] r74156 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter codewriter/test metainterp Message-ID: <20100428111341.31EE4282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:13:39 2010 New Revision: 74156 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: setfield. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed Apr 28 13:13:39 2010 @@ -470,39 +470,63 @@ assert isinstance(arraydescr, Descr) llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) - def do_setfield_gc(self, structbox, newvaluebox, fielddescr): +## def do_setfield_gc(self, structbox, newvaluebox, fielddescr): +## assert isinstance(fielddescr, Descr) +## struct = structbox.getref_base() +## if fielddescr.typeinfo == REF: +## newvalue = newvaluebox.getref_base() +## llimpl.do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) +## elif fielddescr.typeinfo == INT: +## newvalue = newvaluebox.getint() +## llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue, +## self.memo_cast) +## elif fielddescr.typeinfo == FLOAT: +## newvalue = newvaluebox.getfloat() +## llimpl.do_setfield_gc_float(struct, fielddescr.ofs, newvalue) +## else: +## raise NotImplementedError + + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) - struct = structbox.getref_base() - if fielddescr.typeinfo == REF: - newvalue = newvaluebox.getref_base() - llimpl.do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - newvalue = newvaluebox.getint() - llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue, - self.memo_cast) - elif fielddescr.typeinfo == FLOAT: - newvalue = newvaluebox.getfloat() - llimpl.do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError + llimpl.do_setfield_gc_int(struct, fielddescr.ofs, newvalue) + bh_setfield_gc_c = bh_setfield_gc_i + bh_setfield_gc_u = bh_setfield_gc_i + def bh_setfield_gc_r(self, struct, fielddescr, newvalue): + assert isinstance(fielddescr, Descr) + llimpl.do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) + def bh_setfield_gc_f(self, struct, fielddescr, newvalue): + assert isinstance(fielddescr, Descr) + llimpl.do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - def do_setfield_raw(self, structbox, newvaluebox, fielddescr): +## def do_setfield_raw(self, structbox, newvaluebox, fielddescr): +## assert isinstance(fielddescr, Descr) +## struct = self.cast_int_to_adr(structbox.getint()) +## if fielddescr.typeinfo == REF: +## newvalue = newvaluebox.getref_base() +## llimpl.do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue, +## self.memo_cast) +## elif fielddescr.typeinfo == INT: +## newvalue = newvaluebox.getint() +## llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue, +## self.memo_cast) +## elif fielddescr.typeinfo == FLOAT: +## newvalue = newvaluebox.getfloat() +## llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue, +## self.memo_cast) +## else: +## raise NotImplementedError + + def bh_setfield_raw_i(self, struct, fielddescr, newvalue): assert isinstance(fielddescr, Descr) - struct = self.cast_int_to_adr(structbox.getint()) - if fielddescr.typeinfo == REF: - newvalue = newvaluebox.getref_base() - llimpl.do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue, - self.memo_cast) - elif fielddescr.typeinfo == INT: - newvalue = newvaluebox.getint() - llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue, - self.memo_cast) - elif fielddescr.typeinfo == FLOAT: - newvalue = newvaluebox.getfloat() - llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue, - self.memo_cast) - else: - raise NotImplementedError + llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue) + bh_setfield_raw_c = bh_setfield_raw_i + bh_setfield_raw_u = bh_setfield_raw_i + def bh_setfield_raw_r(self, struct, fielddescr, newvalue): + assert isinstance(fielddescr, Descr) + llimpl.do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) + def bh_setfield_raw_f(self, struct, fielddescr, newvalue): + assert isinstance(fielddescr, Descr) + llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue) def do_same_as(self, box1): return box1.clonebox() Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed Apr 28 13:13:39 2010 @@ -240,9 +240,31 @@ def do_setfield_gc(self, structbox, newvaluebox, fielddescr): raise NotImplementedError + def bh_setfield_gc_i(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_gc_c(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_gc_u(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_gc_r(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_gc_f(self, struct, fielddescr, newvalue): + raise NotImplementedError + def do_setfield_raw(self, structbox, newvaluebox, fielddescr): raise NotImplementedError - + + def bh_setfield_raw_i(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_raw_c(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_raw_u(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_raw_r(self, struct, fielddescr, newvalue): + raise NotImplementedError + def bh_setfield_raw_f(self, struct, fielddescr, newvalue): + raise NotImplementedError + def do_newstr(self, lengthbox): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:13:39 2010 @@ -287,7 +287,30 @@ if self.is_typeptr_getset(op): # ignore the operation completely -- instead, it's done by 'new' raise NoOp - XXX + # turn the flow graph 'setfield' operation into our own version + [v_inst, c_fieldname, v_value] = op.args + RESULT = v_value.concretetype + if RESULT is lltype.Void: + raise NoOp + # check for virtualizable + #if self.is_virtualizable_getset(op): + # vinfo = self.codewriter.metainterp_sd.virtualizable_info + # index = vinfo.static_field_to_extra_box[op.args[1].value] + # self.emit('setfield_vable', + # self.var_position(v_inst), + # index, + # self.var_position(v_value)) + # return + argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') + descr = self.cpu.fielddescrof(v_inst.concretetype.TO, + c_fieldname.value) + if isinstance(RESULT, lltype.Primitive): + kind = primitive_type_size[RESULT] + else: + kind = getkind(RESULT)[0] + return SpaceOperation('setfield_%s_%s' % (argname, kind), + [v_inst, descr, v_value], + None) def is_typeptr_getset(self, op): return (op.args[1].value == 'typeptr' and Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 13:13:39 2010 @@ -181,6 +181,35 @@ assert op1.args == [v_parent] assert op1.result == v_result +def test_setfield(): + # XXX a more compact encoding would be possible; see test_getfield() + S1 = lltype.Struct('S1') + S2 = lltype.GcStruct('S2') + S = lltype.GcStruct('S', ('int', lltype.Signed), + ('ps1', lltype.Ptr(S1)), + ('ps2', lltype.Ptr(S2)), + ('flt', lltype.Float), + ('boo', lltype.Bool), + ('chr', lltype.Char), + ('unc', lltype.UniChar)) + for name, suffix in [('int', 'i'), + ('ps1', 'i'), + ('ps2', 'r'), + ('flt', 'f'), + ('boo', 'c'), + ('chr', 'c'), + ('unc', 'u')]: + v_parent = varoftype(lltype.Ptr(S)) + c_name = Constant(name, lltype.Void) + v_newvalue = varoftype(getattr(S, name)) + op = SpaceOperation('setfield', [v_parent, c_name, v_newvalue], + varoftype(lltype.Void)) + op1 = Transformer(FakeCPU()).rewrite_operation(op) + assert op1.opname == 'setfield_gc_' + suffix + fielddescr = ('fielddescr', S, name) + assert op1.args == [v_parent, fielddescr, v_newvalue] + assert op1.result is None + def test_malloc_new(): S = lltype.GcStruct('S') v = varoftype(lltype.Ptr(S)) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 13:13:39 2010 @@ -505,6 +505,38 @@ opimpl_getfield_raw_r_pure = opimpl_getfield_raw_r opimpl_getfield_raw_f_pure = opimpl_getfield_raw_f + @arguments("r", "d", "i") + def opimpl_setfield_gc_i(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) + @arguments("r", "d", "i") + def opimpl_setfield_gc_c(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) + @arguments("r", "d", "i") + def opimpl_setfield_gc_u(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) + @arguments("r", "d", "r") + def opimpl_setfield_gc_r(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) + @arguments("r", "d", "f") + def opimpl_setfield_gc_f(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) + + @arguments("r", "d", "i") + def opimpl_setfield_raw_i(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) + @arguments("r", "d", "i") + def opimpl_setfield_raw_c(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) + @arguments("r", "d", "i") + def opimpl_setfield_raw_u(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) + @arguments("r", "d", "r") + def opimpl_setfield_raw_r(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) + @arguments("r", "d", "f") + def opimpl_setfield_raw_f(self, struct, fielddescr, newvalue): + self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) + @arguments("d", returns="r") def opimpl_new_with_vtable(self, descr): return self.cpu.bh_new_with_vtable(descr) From arigo at codespeak.net Wed Apr 28 13:19:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:19:10 +0200 (CEST) Subject: [pypy-svn] r74157 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100428111910.303C5282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:19:08 2010 New Revision: 74157 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Tweaks. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:19:08 2010 @@ -136,6 +136,7 @@ def rewrite_op_cast_int_to_unichar(self, op): raise NoOp def rewrite_op_cast_char_to_int(self, op): raise NoOp def rewrite_op_cast_unichar_to_int(self, op): raise NoOp + def rewrite_op_cast_bool_to_int(self, op): raise NoOp def rewrite_op_cast_pointer(self, op): raise NoOp def rewrite_op_direct_call(self, op): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 13:19:08 2010 @@ -281,6 +281,14 @@ return a & b @arguments("i", "i", returns="i") + def opimpl_int_or(self, a, b): + return a | b + + @arguments("i", "i", returns="i") + def opimpl_int_xor(self, a, b): + return a ^ b + + @arguments("i", "i", returns="i") def opimpl_int_floordiv(self, a, b): return llop.int_floordiv(lltype.Signed, a, b) @@ -293,6 +301,25 @@ c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) return intmask(c) + @arguments("i", "i", returns="i") + def opimpl_int_lt(self, a, b): + return int(a < b) + @arguments("i", "i", returns="i") + def opimpl_int_le(self, a, b): + return int(a <= b) + @arguments("i", "i", returns="i") + def opimpl_int_eq(self, a, b): + return int(a == b) + @arguments("i", "i", returns="i") + def opimpl_int_ne(self, a, b): + return int(a != b) + @arguments("i", "i", returns="i") + def opimpl_int_gt(self, a, b): + return int(a > b) + @arguments("i", "i", returns="i") + def opimpl_int_ge(self, a, b): + return int(a >= b) + @arguments("i") def opimpl_int_return(self, a): self.result_i = a @@ -424,9 +451,9 @@ @arguments("i", "d", "R", returns="f") def opimpl_residual_call_r_f(self, func, calldescr, args_r): return self.cpu.bh_call_f(func, calldescr, None, args_r, None) - @arguments("i", "d", "R", returns="v") + @arguments("i", "d", "R") def opimpl_residual_call_r_v(self, func, calldescr, args_r): - return self.cpu.bh_call_v(func, calldescr, None, args_r, None) + self.cpu.bh_call_v(func, calldescr, None, args_r, None) @arguments("i", "d", "I", "R", returns="i") def opimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): @@ -437,9 +464,9 @@ @arguments("i", "d", "I", "R", returns="f") def opimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, None) - @arguments("i", "d", "I", "R", returns="v") + @arguments("i", "d", "I", "R") def opimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): - return self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) + self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) @arguments("i", "d", "I", "R", "F", returns="i") def opimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): @@ -450,9 +477,9 @@ @arguments("i", "d", "I", "R", "F", returns="f") def opimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) - @arguments("i", "d", "I", "R", "F", returns="v") + @arguments("i", "d", "I", "R", "F") def opimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): - return self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) + self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) @arguments("d", "i", returns="r") def opimpl_new_array(self, arraydescr, length): From arigo at codespeak.net Wed Apr 28 13:22:44 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:22:44 +0200 (CEST) Subject: [pypy-svn] r74158 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter metainterp Message-ID: <20100428112244.1A467282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:22:42 2010 New Revision: 74158 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: indirect_call, new. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed Apr 28 13:22:42 2010 @@ -428,6 +428,10 @@ assert isinstance(size, Descr) return history.BoxPtr(llimpl.do_new(size.ofs)) + def bh_new(self, sizedescr): + assert isinstance(sizedescr, Descr) + return llimpl.do_new(sizedescr.ofs) + def bh_new_with_vtable(self, sizevtabledescr): result = llimpl.do_new(sizevtabledescr.ofs) llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed Apr 28 13:22:42 2010 @@ -209,6 +209,9 @@ def do_new(self, sizedescr): raise NotImplementedError + def bh_new(self, sizedescr): + raise NotImplementedError + def do_new_with_vtable(self, classbox): raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:22:42 2010 @@ -163,6 +163,10 @@ [op.args[0], calldescr] + sublists, op.result) + def rewrite_op_indirect_call(self, op): + op1 = SpaceOperation('direct_call', op.args[:-1], op.result) + return self.rewrite_op_direct_call(op1) + def add_in_correct_list(self, v, lst_i, lst_r, lst_f): kind = getkind(v.concretetype) if kind == 'void': return Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 13:22:42 2010 @@ -565,6 +565,10 @@ self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) @arguments("d", returns="r") + def opimpl_new(self, descr): + return self.cpu.bh_new(descr) + + @arguments("d", returns="r") def opimpl_new_with_vtable(self, descr): return self.cpu.bh_new_with_vtable(descr) From arigo at codespeak.net Wed Apr 28 13:23:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:23:37 +0200 (CEST) Subject: [pypy-svn] r74159 - pypy/branch/blackhole-improvement/pypy/jit/codewriter Message-ID: <20100428112337.E1A62282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:23:36 2010 New Revision: 74159 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Log: gc_identityhash. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:23:36 2010 @@ -194,6 +194,8 @@ rewrite_op_int_mod_ovf = _do_builtin_call rewrite_op_int_mod_zer = _do_builtin_call + rewrite_op_gc_identityhash = _do_builtin_call + def rewrite_op_hint(self, op): hints = op.args[1].value if hints.get('promote') and op.args[0].concretetype is not lltype.Void: From arigo at codespeak.net Wed Apr 28 13:37:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:37:47 +0200 (CEST) Subject: [pypy-svn] r74160 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428113747.5CE52282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:37:45 2010 New Revision: 74160 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: ptr_eq, ptr_ne, ptr_iszero, ptr_nonzero, and the same on exitswitches. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:37:45 2010 @@ -99,7 +99,9 @@ return False # variable is also used in cur block if v is op.result: if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', - 'int_gt', 'int_ge', 'int_is_true'): + 'int_gt', 'int_ge', 'int_is_true', + 'ptr_eq', 'ptr_ne', + 'ptr_iszero', 'ptr_nonzero'): return False # not a supported operation # ok! optimize this case block.operations.remove(op) @@ -117,6 +119,13 @@ link.llexitcase = link.exitcase = not link.exitcase opname = 'int_is_true' args = [args[0]] + elif op.opname in ('ptr_eq', 'ptr_ne'): + if isinstance(args[0], Constant): + args = args[::-1] + if isinstance(args[1], Constant) and not args[1].value: + opname = {'ptr_eq': 'ptr_iszero', + 'ptr_ne': 'ptr_nonzero'}[op.opname] + args = [args[0]] block.exitswitch = (opname,) + tuple(args) return True return False @@ -352,6 +361,21 @@ sizedescr = self.cpu.sizeof(STRUCT) return SpaceOperation('new', [sizedescr], op.result) + def _rewrite_op_ptr_eq(self, op, opname): + arg0, arg1 = op.args + if isinstance(arg0, Constant) and not arg0.value: + return SpaceOperation(opname, [arg1], op.result) + elif isinstance(arg1, Constant) and not arg1.value: + return SpaceOperation(opname, [arg0], op.result) + else: + return op + + def rewrite_op_ptr_eq(self, op): + return self._rewrite_op_ptr_eq(op, 'ptr_iszero') + + def rewrite_op_ptr_ne(self, op): + return self._rewrite_op_ptr_eq(op, 'ptr_nonzero') + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 13:37:45 2010 @@ -95,6 +95,54 @@ assert exits[0].exitcase == exits[0].llexitcase == linkcase2 assert exits[1].exitcase == exits[1].llexitcase == linkcase1 +def test_optimize_goto_if_not__ptr_eq(): + for opname in ['ptr_eq', 'ptr_ne']: + v1 = Variable() + v2 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + block = Block([v1, v2]) + block.operations = [SpaceOperation(opname, [v1, v2], v3)] + block.exitswitch = v3 + block.exits = exits = [FakeLink(False), FakeLink(True)] + res = Transformer().optimize_goto_if_not(block) + assert res == True + assert block.operations == [] + assert block.exitswitch == (opname, v1, v2) + assert block.exits == exits + +def test_optimize_goto_if_not__ptr_iszero(): + for opname in ['ptr_iszero', 'ptr_nonzero']: + v1 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + block = Block([v1]) + block.operations = [SpaceOperation(opname, [v1], v3)] + block.exitswitch = v3 + block.exits = exits = [FakeLink(False), FakeLink(True)] + res = Transformer().optimize_goto_if_not(block) + assert res == True + assert block.operations == [] + assert block.exitswitch == (opname, v1) + assert block.exits == exits + +def test_optimize_goto_if_not__ptr_eq_reduced(): + c0 = Constant(lltype.nullptr(rclass.OBJECT), rclass.OBJECTPTR) + for opname, reducedname in [('ptr_eq', 'ptr_iszero'), + ('ptr_ne', 'ptr_nonzero')]: + for nullindex in [0, 1]: + v1 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + block = Block([v1]) + args = [v1, v1] + args[nullindex] = c0 + block.operations = [SpaceOperation(opname, args, v3)] + block.exitswitch = v3 + block.exits = exits = [FakeLink(False), FakeLink(True)] + res = Transformer().optimize_goto_if_not(block) + assert res == True + assert block.operations == [] + assert block.exitswitch == (reducedname, v1) + assert block.exits == exits + def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: @@ -258,3 +306,26 @@ assert block.operations == [] assert block.exits[0].target is block2 assert block.exits[0].args == [v1] + +def test_ptr_eq(): + v1 = varoftype(rclass.OBJECTPTR) + v2 = varoftype(rclass.OBJECTPTR) + v3 = varoftype(lltype.Bool) + c0 = Constant(lltype.nullptr(rclass.OBJECT), rclass.OBJECTPTR) + # + for opname, reducedname in [('ptr_eq', 'ptr_iszero'), + ('ptr_ne', 'ptr_nonzero')]: + op = SpaceOperation(opname, [v1, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == opname + assert op1.args == [v1, v2] + # + op = SpaceOperation(opname, [v1, c0], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v1] + # + op = SpaceOperation(opname, [c0, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v2] From arigo at codespeak.net Wed Apr 28 13:57:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:57:41 +0200 (CEST) Subject: [pypy-svn] r74161 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428115741.5C65B282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:57:39 2010 New Revision: 74161 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Uniformize the reductions: int_eq(x,0) => int_is_zero(x) int_ne(x,0) => int_is_true(x) ptr_eq(x,0) => ptr_iszero(x) ptr_ne(x,0) => ptr_nonzero(x) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 13:57:39 2010 @@ -105,28 +105,7 @@ return False # not a supported operation # ok! optimize this case block.operations.remove(op) - opname = op.opname - args = op.args - if op.opname in ('int_ne', 'int_eq'): - if isinstance(args[0], Constant): - args = args[::-1] - if isinstance(args[1], Constant) and args[1].value == 0: - if opname == 'int_eq': - # must invert the two exit links - link = block.exits[0] - link.llexitcase = link.exitcase = not link.exitcase - link = block.exits[1] - link.llexitcase = link.exitcase = not link.exitcase - opname = 'int_is_true' - args = [args[0]] - elif op.opname in ('ptr_eq', 'ptr_ne'): - if isinstance(args[0], Constant): - args = args[::-1] - if isinstance(args[1], Constant) and not args[1].value: - opname = {'ptr_eq': 'ptr_iszero', - 'ptr_ne': 'ptr_nonzero'}[op.opname] - args = [args[0]] - block.exitswitch = (opname,) + tuple(args) + block.exitswitch = (op.opname,) + tuple(op.args) return True return False @@ -361,7 +340,7 @@ sizedescr = self.cpu.sizeof(STRUCT) return SpaceOperation('new', [sizedescr], op.result) - def _rewrite_op_ptr_eq(self, op, opname): + def _rewrite_equality(self, op, opname): arg0, arg1 = op.args if isinstance(arg0, Constant) and not arg0.value: return SpaceOperation(opname, [arg1], op.result) @@ -370,11 +349,17 @@ else: return op + def rewrite_op_int_eq(self, op): + return self._rewrite_equality(op, 'int_is_zero') + + def rewrite_op_int_ne(self, op): + return self._rewrite_equality(op, 'int_is_true') + def rewrite_op_ptr_eq(self, op): - return self._rewrite_op_ptr_eq(op, 'ptr_iszero') + return self._rewrite_equality(op, 'ptr_iszero') def rewrite_op_ptr_ne(self, op): - return self._rewrite_op_ptr_eq(op, 'ptr_nonzero') + return self._rewrite_equality(op, 'ptr_nonzero') # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 13:57:39 2010 @@ -69,32 +69,6 @@ block.exits = [FakeLink(False), FakeLink(True)] assert not Transformer().optimize_goto_if_not(block) -def test_optimize_goto_if_not__int_ne(): - c0 = Constant(0, lltype.Signed) - v1 = Variable() - v3 = Variable(); v3.concretetype = lltype.Bool - for linkcase1 in [False, True]: - linkcase2 = not linkcase1 - for op, args in [('int_ne', [v1, c0]), - ('int_ne', [c0, v1]), - ('int_eq', [v1, c0]), - ('int_eq', [c0, v1])]: - block = Block([v1]) - block.operations = [SpaceOperation(op, args, v3)] - block.exitswitch = v3 - block.exits = exits = [FakeLink(linkcase1), FakeLink(linkcase2)] - res = Transformer().optimize_goto_if_not(block) - assert res == True - assert block.operations == [] - assert block.exitswitch == ('int_is_true', v1) - assert block.exits == exits - if op == 'int_ne': - assert exits[0].exitcase == exits[0].llexitcase == linkcase1 - assert exits[1].exitcase == exits[1].llexitcase == linkcase2 - else: - assert exits[0].exitcase == exits[0].llexitcase == linkcase2 - assert exits[1].exitcase == exits[1].llexitcase == linkcase1 - def test_optimize_goto_if_not__ptr_eq(): for opname in ['ptr_eq', 'ptr_ne']: v1 = Variable() @@ -124,25 +98,6 @@ assert block.exitswitch == (opname, v1) assert block.exits == exits -def test_optimize_goto_if_not__ptr_eq_reduced(): - c0 = Constant(lltype.nullptr(rclass.OBJECT), rclass.OBJECTPTR) - for opname, reducedname in [('ptr_eq', 'ptr_iszero'), - ('ptr_ne', 'ptr_nonzero')]: - for nullindex in [0, 1]: - v1 = Variable() - v3 = Variable(); v3.concretetype = lltype.Bool - block = Block([v1]) - args = [v1, v1] - args[nullindex] = c0 - block.operations = [SpaceOperation(opname, args, v3)] - block.exitswitch = v3 - block.exits = exits = [FakeLink(False), FakeLink(True)] - res = Transformer().optimize_goto_if_not(block) - assert res == True - assert block.operations == [] - assert block.exitswitch == (reducedname, v1) - assert block.exits == exits - def test_residual_call(): for RESTYPE in [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void]: @@ -307,6 +262,29 @@ assert block.exits[0].target is block2 assert block.exits[0].args == [v1] +def test_int_eq(): + v1 = varoftype(lltype.Signed) + v2 = varoftype(lltype.Signed) + v3 = varoftype(lltype.Bool) + c0 = Constant(0, lltype.Signed) + # + for opname, reducedname in [('int_eq', 'int_is_zero'), + ('int_ne', 'int_is_true')]: + op = SpaceOperation(opname, [v1, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == opname + assert op1.args == [v1, v2] + # + op = SpaceOperation(opname, [v1, c0], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v1] + # + op = SpaceOperation(opname, [c0, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v2] + def test_ptr_eq(): v1 = varoftype(rclass.OBJECTPTR) v2 = varoftype(rclass.OBJECTPTR) @@ -329,3 +307,26 @@ op1 = Transformer().rewrite_operation(op) assert op1.opname == reducedname assert op1.args == [v2] + +def test_nongc_ptr_eq(): + v1 = varoftype(rclass.NONGCOBJECTPTR) + v2 = varoftype(rclass.NONGCOBJECTPTR) + v3 = varoftype(lltype.Bool) + c0 = Constant(lltype.nullptr(rclass.NONGCOBJECT), rclass.NONGCOBJECTPTR) + # + for opname, reducedname in [('int_eq', 'int_is_zero'), + ('int_ne', 'int_is_true')]: + op = SpaceOperation(opname, [v1, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == opname + assert op1.args == [v1, v2] + # + op = SpaceOperation(opname, [v1, c0], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v1] + # + op = SpaceOperation(opname, [c0, v2], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == reducedname + assert op1.args == [v2] From arigo at codespeak.net Wed Apr 28 13:58:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 13:58:48 +0200 (CEST) Subject: [pypy-svn] r74162 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100428115848.41361282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 13:58:46 2010 New Revision: 74162 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Fixes. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 13:58:46 2010 @@ -86,12 +86,16 @@ argcode = argcodes[next_argcode] next_argcode = next_argcode + 1 if argcode == 'i': + assert argtype == 'i' value = self.registers_i[ord(code[position])] elif argcode == 'c': + assert argtype == 'i' value = signedord(code[position]) elif argcode == 'r': + assert argtype == 'r' value = self.registers_r[ord(code[position])] elif argcode == 'f': + assert argtype == 'f' value = self.registers_f[ord(code[position])] else: raise AssertionError("bad argcode") @@ -320,6 +324,19 @@ def opimpl_int_ge(self, a, b): return int(a >= b) + @arguments("r", "r", returns="i") + def opimpl_ptr_eq(self, a, b): + return int(a == b) + @arguments("r", "r", returns="i") + def opimpl_ptr_ne(self, a, b): + return int(a != b) + @arguments("r", returns="i") + def opimpl_ptr_iszero(self, a): + return int(not a) + @arguments("r", returns="i") + def opimpl_ptr_nonzero(self, a): + return int(bool(a)) + @arguments("i") def opimpl_int_return(self, a): self.result_i = a @@ -390,6 +407,34 @@ else: return target + @arguments("L", "r", "r", "pc", returns="L") + def opimpl_goto_if_not_ptr_eq(self, target, a, b, pc): + if a == b: + return pc + else: + return target + + @arguments("L", "r", "r", "pc", returns="L") + def opimpl_goto_if_not_ptr_ne(self, target, a, b, pc): + if a != b: + return pc + else: + return target + + @arguments("L", "r", "pc", returns="L") + def opimpl_goto_if_not_ptr_iszero(self, target, a, pc): + if not a: + return pc + else: + return target + + @arguments("L", "r", "pc", returns="L") + def opimpl_goto_if_not_ptr_nonzero(self, target, a, pc): + if a: + return pc + else: + return target + @arguments("L", returns="L") def opimpl_goto(self, target): return target @@ -510,19 +555,19 @@ opimpl_getfield_gc_r_pure = opimpl_getfield_gc_r opimpl_getfield_gc_f_pure = opimpl_getfield_gc_f - @arguments("r", "d", returns="i") + @arguments("i", "d", returns="i") def opimpl_getfield_raw_i(self, struct, fielddescr): return self.cpu.bh_getfield_raw_i(struct, fielddescr) - @arguments("r", "d", returns="i") + @arguments("i", "d", returns="i") def opimpl_getfield_raw_c(self, struct, fielddescr): return self.cpu.bh_getfield_raw_c(struct, fielddescr) - @arguments("r", "d", returns="i") + @arguments("i", "d", returns="i") def opimpl_getfield_raw_u(self, struct, fielddescr): return self.cpu.bh_getfield_raw_u(struct, fielddescr) - @arguments("r", "d", returns="r") + @arguments("i", "d", returns="r") def opimpl_getfield_raw_r(self, struct, fielddescr): return self.cpu.bh_getfield_raw_r(struct, fielddescr) - @arguments("r", "d", returns="f") + @arguments("i", "d", returns="f") def opimpl_getfield_raw_f(self, struct, fielddescr): return self.cpu.bh_getfield_raw_f(struct, fielddescr) @@ -548,19 +593,19 @@ def opimpl_setfield_gc_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) - @arguments("r", "d", "i") + @arguments("i", "d", "i") def opimpl_setfield_raw_i(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) - @arguments("r", "d", "i") + @arguments("i", "d", "i") def opimpl_setfield_raw_c(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) - @arguments("r", "d", "i") + @arguments("i", "d", "i") def opimpl_setfield_raw_u(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) - @arguments("r", "d", "r") + @arguments("i", "d", "r") def opimpl_setfield_raw_r(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) - @arguments("r", "d", "f") + @arguments("i", "d", "f") def opimpl_setfield_raw_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) From agaynor at codespeak.net Wed Apr 28 13:58:49 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 28 Apr 2010 13:58:49 +0200 (CEST) Subject: [pypy-svn] r74163 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20100428115849.5C522282B9D@codespeak.net> Author: agaynor Date: Wed Apr 28 13:58:47 2010 New Revision: 74163 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: Fixed app tests, count could return -1 in some cases. Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Wed Apr 28 13:58:47 2010 @@ -564,7 +564,12 @@ elif m == 1: return cls.ll_count_char(s1, s2.chars[0], start, end) - return cls.ll_search(s1, s2, start, end, FAST_COUNT) + res = cls.ll_search(s1, s2, start, end, FAST_COUNT) + # For a few cases ll_search can return -1 to indicate an "impossible" + # condition for a string match, count just returns 0 in these cases. + if res < 0: + res = 0 + return res @purefunction def ll_search(s1, s2, start, end, mode): From arigo at codespeak.net Wed Apr 28 14:08:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 14:08:42 +0200 (CEST) Subject: [pypy-svn] r74164 - pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem Message-ID: <20100428120842.BFBD5282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 14:08:41 2010 New Revision: 74164 Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rclass.py Log: Argh! Typo. No clue how it did not already trigger crashes. Modified: pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/blackhole-improvement/pypy/rpython/lltypesystem/rclass.py Wed Apr 28 14:08:41 2010 @@ -72,7 +72,7 @@ hints = {'immutable': True})) # non-gc case NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE)) -NONGCOBJECTPTR = Ptr(OBJECT) +NONGCOBJECTPTR = Ptr(NONGCOBJECT) OBJECT_BY_FLAVOR = {'gc': OBJECT, 'raw': NONGCOBJECT} From arigo at codespeak.net Wed Apr 28 14:09:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 14:09:24 +0200 (CEST) Subject: [pypy-svn] r74165 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428120924.E5B3B282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 14:09:23 2010 New Revision: 74165 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Log: Correctly transform ptr_xxx into int_xxx operations when the pointers being handled are non-gc. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 14:09:23 2010 @@ -349,6 +349,16 @@ else: return op + def _rewrite_nongc_ptrs(self, op): + if op.args[0].concretetype.TO._gckind == 'gc': + return op + else: + opname = {'ptr_eq': 'int_eq', + 'ptr_ne': 'int_ne', + 'ptr_iszero': 'int_is_zero', + 'ptr_nonzero': 'int_is_true'}[op.opname] + return SpaceOperation(opname, op.args, op.result) + def rewrite_op_int_eq(self, op): return self._rewrite_equality(op, 'int_is_zero') @@ -356,10 +366,15 @@ return self._rewrite_equality(op, 'int_is_true') def rewrite_op_ptr_eq(self, op): - return self._rewrite_equality(op, 'ptr_iszero') + op1 = self._rewrite_equality(op, 'ptr_iszero') + return self._rewrite_nongc_ptrs(op1) def rewrite_op_ptr_ne(self, op): - return self._rewrite_equality(op, 'ptr_nonzero') + op1 = self._rewrite_equality(op, 'ptr_nonzero') + return self._rewrite_nongc_ptrs(op1) + + rewrite_op_ptr_iszero = _rewrite_nongc_ptrs + rewrite_op_ptr_nonzero = _rewrite_nongc_ptrs # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Wed Apr 28 14:09:23 2010 @@ -314,11 +314,11 @@ v3 = varoftype(lltype.Bool) c0 = Constant(lltype.nullptr(rclass.NONGCOBJECT), rclass.NONGCOBJECTPTR) # - for opname, reducedname in [('int_eq', 'int_is_zero'), - ('int_ne', 'int_is_true')]: + for opname, reducedname in [('ptr_eq', 'int_is_zero'), + ('ptr_ne', 'int_is_true')]: op = SpaceOperation(opname, [v1, v2], v3) op1 = Transformer().rewrite_operation(op) - assert op1.opname == opname + assert op1.opname == opname.replace('ptr_', 'int_') assert op1.args == [v1, v2] # op = SpaceOperation(opname, [v1, c0], v3) @@ -330,3 +330,13 @@ op1 = Transformer().rewrite_operation(op) assert op1.opname == reducedname assert op1.args == [v2] + # + op = SpaceOperation('ptr_iszero', [v1], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'int_is_zero' + assert op1.args == [v1] + # + op = SpaceOperation('ptr_nonzero', [v1], v3) + op1 = Transformer().rewrite_operation(op) + assert op1.opname == 'int_is_true' + assert op1.args == [v1] From arigo at codespeak.net Wed Apr 28 14:27:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 14:27:24 +0200 (CEST) Subject: [pypy-svn] r74166 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter metainterp metainterp/test Message-ID: <20100428122724.D81A2282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 14:27:23 2010 New Revision: 74166 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: cast_ptr_to_int and a few other missing operations in blackhole.py. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Wed Apr 28 14:27:23 2010 @@ -585,6 +585,9 @@ return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) + def bh_cast_ptr_to_int(self, ptr): + return llimpl.cast_to_int(ptr) + def force(self, force_token): token = self.cast_int_to_adr(force_token) frame = llimpl.get_forced_token_frame(token) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Wed Apr 28 14:27:23 2010 @@ -305,6 +305,9 @@ def do_cast_ptr_to_int(self, ptrbox): raise NotImplementedError + def bh_cast_ptr_to_int(self, ptr): + raise NotImplementedError + def do_call_may_force(self, args, calldescr): return self.do_call(args, calldescr) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 14:27:23 2010 @@ -349,8 +349,11 @@ else: return op + def _is_gc(self, v): + return v.concretetype.TO._gckind == 'gc' + def _rewrite_nongc_ptrs(self, op): - if op.args[0].concretetype.TO._gckind == 'gc': + if self._is_gc(op.args[0]): return op else: opname = {'ptr_eq': 'int_eq', @@ -376,6 +379,12 @@ rewrite_op_ptr_iszero = _rewrite_nongc_ptrs rewrite_op_ptr_nonzero = _rewrite_nongc_ptrs + def rewrite_op_cast_ptr_to_int(self, op): + if self._is_gc(op.args[0]): + return op + else: + raise NoOp + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 14:27:23 2010 @@ -220,6 +220,12 @@ def setarg_i(self, index, value): self.registers_i[index] = value + def setarg_r(self, index, value): + self.registers_r[index] = value + + def setarg_f(self, index, value): + self.registers_f[index] = value + def run(self, jitcode, position): self.copy_constants(self.registers_i, jitcode.constants_i) self.copy_constants(self.registers_r, jitcode.constants_r) @@ -323,6 +329,12 @@ @arguments("i", "i", returns="i") def opimpl_int_ge(self, a, b): return int(a >= b) + @arguments("i", returns="i") + def opimpl_int_is_zero(self, a): + return int(not a) + @arguments("i", returns="i") + def opimpl_int_is_true(self, a): + return int(bool(a)) @arguments("r", "r", returns="i") def opimpl_ptr_eq(self, a, b): @@ -401,6 +413,13 @@ return target @arguments("L", "i", "pc", returns="L") + def opimpl_goto_if_not_int_is_zero(self, target, a, pc): + if not a: + return pc + else: + return target + + @arguments("L", "i", "pc", returns="L") def opimpl_goto_if_not_int_is_true(self, target, a, pc): if a: return pc @@ -620,3 +639,7 @@ @arguments("r", returns="i") def opimpl_classof(self, struct): return self.cpu.bh_classof(struct) + + @arguments("r", returns="i") + def opimpl_cast_ptr_to_int(self, p): + return self.cpu.bh_cast_ptr_to_int(p) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed Apr 28 14:27:23 2010 @@ -9,7 +9,7 @@ from pypy import conftest from pypy.rlib.rarithmetic import ovfcheck from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype def _get_jitcodes(CPUClass, func, values, type_system): @@ -27,8 +27,20 @@ from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder blackholeinterpbuilder = BlackholeInterpBuilder(cw) blackholeinterp = blackholeinterpbuilder.acquire_interp() - for i, value in enumerate(args): - blackholeinterp.setarg_i(i, value) + count_i = count_r = count_f = 0 + for value in args: + T = lltype.typeOf(value) + if T == lltype.Signed: + blackholeinterp.setarg_i(count_i, value) + count_i += 1 + elif T == llmemory.GCREF: + blackholeinterp.setarg_r(count_r, value) + count_r += 1 + elif T == lltype.Float: + blackholeinterp.setarg_f(count_f, value) + count_f += 1 + else: + raise TypeError(value) blackholeinterp.run(mainjitcode, 0) return blackholeinterp.result_i @@ -1477,17 +1489,29 @@ py.test.skip("test written in a style that " "means it's frontend only") from pypy.rpython.lltypesystem import lltype, llmemory - - TP = lltype.GcStruct('x') + + TP = lltype.GcStruct('S1') def f(p): n = lltype.cast_ptr_to_int(p) return n - x = lltype.malloc(TP) - res = self.interp_operations(f, [x]) - expected = self.metainterp.cpu.do_cast_ptr_to_int( - history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value - assert res == expected + xref = lltype.cast_opaque_ptr(llmemory.GCREF, x) + res = self.interp_operations(f, [xref]) + y = llmemory.cast_int_to_adr(res) + y = llmemory.cast_adr_to_ptr(y, lltype.Ptr(TP)) + assert x == y + # + TP = lltype.Struct('S2') + prebuilt = [lltype.malloc(TP, immortal=True), + lltype.malloc(TP, immortal=True)] + def f(x): + p = prebuilt[x] + n = lltype.cast_ptr_to_int(p) + return n + res = self.interp_operations(f, [1]) + y = llmemory.cast_int_to_adr(res) + y = llmemory.cast_adr_to_ptr(y, lltype.Ptr(TP)) + assert prebuilt[1] == y def test_collapsing_ptr_eq(self): S = lltype.GcStruct('S') From arigo at codespeak.net Wed Apr 28 14:33:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 14:33:16 +0200 (CEST) Subject: [pypy-svn] r74167 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100428123316.051E6282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 14:33:15 2010 New Revision: 74167 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Tweaks. Now all of test_basic either passes or is skipped. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Wed Apr 28 14:33:15 2010 @@ -126,6 +126,8 @@ def rewrite_op_cast_unichar_to_int(self, op): raise NoOp def rewrite_op_cast_bool_to_int(self, op): raise NoOp def rewrite_op_cast_pointer(self, op): raise NoOp + def rewrite_op_cast_int_to_uint(self, op): raise NoOp + def rewrite_op_cast_uint_to_int(self, op): raise NoOp def rewrite_op_direct_call(self, op): """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)' Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 14:33:15 2010 @@ -180,8 +180,12 @@ assert next_argcode == len(argcodes) return position # - verbose = self.verbose + # Get the opimpl_xxx method. If we get an AttributeError here, + # it means that either the implementation is missing, or that it + # should not appear here at all but instead be transformed away + # by codewriter/jitter.py. unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name) + verbose = self.verbose argtypes = unrolling_iterable(unboundmethod.argtypes) resulttype = unboundmethod.resulttype handler = func_with_new_name(handler, 'handler_' + name) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed Apr 28 14:33:15 2010 @@ -40,7 +40,7 @@ blackholeinterp.setarg_f(count_f, value) count_f += 1 else: - raise TypeError(value) + raise TypeError(T) blackholeinterp.run(mainjitcode, 0) return blackholeinterp.result_i @@ -246,9 +246,11 @@ from pypy.rlib.rarithmetic import r_uint def f(a, b): + a = r_uint(a) + b = r_uint(b) return a/b - res = self.interp_operations(f, [r_uint(4), r_uint(3)]) + res = self.interp_operations(f, [4, 3]) assert res == 1 def test_direct_call(self): @@ -1473,16 +1475,17 @@ from pypy.rpython.lltypesystem import lltype TP = lltype.Struct('x') - def f(p1, p2): + def f(i1, i2): + p1 = prebuilt[i1] + p2 = prebuilt[i2] a = p1 is p2 b = p1 is not p2 c = bool(p1) d = not bool(p2) return 1000*a + 100*b + 10*c + d - x = lltype.malloc(TP, flavor='raw') - expected = f(x, x) - assert self.interp_operations(f, [x, x]) == expected - lltype.free(x, flavor='raw') + prebuilt = [lltype.malloc(TP, flavor='raw')] * 2 + expected = f(0, 1) + assert self.interp_operations(f, [0, 1]) == expected def test_casts(self): if not self.basic: From afa at codespeak.net Wed Apr 28 14:55:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 14:55:43 +0200 (CEST) Subject: [pypy-svn] r74170 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100428125543.64B8F282B9D@codespeak.net> Author: afa Date: Wed Apr 28 14:55:42 2010 New Revision: 74170 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Log: Fix test_InitModule4Dotted on Windows Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_cpyext.py Wed Apr 28 14:55:42 2010 @@ -51,7 +51,7 @@ def compile_module(modname, **kwds): eci = ExternalCompilationInfo( - export_symbols=['init%s' % (modname,)], + export_symbols=['init%s' % (modname.split('.')[-1],)], include_dirs=api.include_dirs, **kwds ) From arigo at codespeak.net Wed Apr 28 14:56:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 14:56:22 +0200 (CEST) Subject: [pypy-svn] r74171 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100428125622.52F2C282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 14:56:20 2010 New Revision: 74171 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Log: Don't put the 'self' argument at all on the operations that don't need it. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Wed Apr 28 14:56:20 2010 @@ -119,6 +119,8 @@ elif argtype == 'F': reg = self.registers_f[index] value.append(reg) position += length + elif argtype == 'self': + value = self elif argtype == 'pc': value = position elif argtype == 'd': @@ -136,7 +138,7 @@ # call the method opimpl_xxx() try: - result = unboundmethod(self, *args) + result = unboundmethod(*args) except Exception, e: if verbose and not we_are_translated(): print '-> %s!' % (e.__class__.__name__,) @@ -184,7 +186,7 @@ # it means that either the implementation is missing, or that it # should not appear here at all but instead be transformed away # by codewriter/jitter.py. - unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name) + unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name).im_func verbose = self.verbose argtypes = unrolling_iterable(unboundmethod.argtypes) resulttype = unboundmethod.resulttype @@ -279,95 +281,95 @@ # ---------- @arguments("i", "i", returns="i") - def opimpl_int_add(self, a, b): + def opimpl_int_add(a, b): return a + b @arguments("i", "i", returns="i") - def opimpl_int_sub(self, a, b): + def opimpl_int_sub(a, b): return a - b @arguments("i", "i", returns="i") - def opimpl_int_mul(self, a, b): + def opimpl_int_mul(a, b): return a * b @arguments("i", "i", returns="i") - def opimpl_int_and(self, a, b): + def opimpl_int_and(a, b): return a & b @arguments("i", "i", returns="i") - def opimpl_int_or(self, a, b): + def opimpl_int_or(a, b): return a | b @arguments("i", "i", returns="i") - def opimpl_int_xor(self, a, b): + def opimpl_int_xor(a, b): return a ^ b @arguments("i", "i", returns="i") - def opimpl_int_floordiv(self, a, b): + def opimpl_int_floordiv(a, b): return llop.int_floordiv(lltype.Signed, a, b) @arguments("i", "i", returns="i") - def opimpl_int_mod(self, a, b): + def opimpl_int_mod(a, b): return llop.int_mod(lltype.Signed, a, b) @arguments("i", "i", returns="i") - def opimpl_uint_floordiv(self, a, b): + def opimpl_uint_floordiv(a, b): c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) return intmask(c) @arguments("i", "i", returns="i") - def opimpl_int_lt(self, a, b): + def opimpl_int_lt(a, b): return int(a < b) @arguments("i", "i", returns="i") - def opimpl_int_le(self, a, b): + def opimpl_int_le(a, b): return int(a <= b) @arguments("i", "i", returns="i") - def opimpl_int_eq(self, a, b): + def opimpl_int_eq(a, b): return int(a == b) @arguments("i", "i", returns="i") - def opimpl_int_ne(self, a, b): + def opimpl_int_ne(a, b): return int(a != b) @arguments("i", "i", returns="i") - def opimpl_int_gt(self, a, b): + def opimpl_int_gt(a, b): return int(a > b) @arguments("i", "i", returns="i") - def opimpl_int_ge(self, a, b): + def opimpl_int_ge(a, b): return int(a >= b) @arguments("i", returns="i") - def opimpl_int_is_zero(self, a): + def opimpl_int_is_zero(a): return int(not a) @arguments("i", returns="i") - def opimpl_int_is_true(self, a): + def opimpl_int_is_true(a): return int(bool(a)) @arguments("r", "r", returns="i") - def opimpl_ptr_eq(self, a, b): + def opimpl_ptr_eq(a, b): return int(a == b) @arguments("r", "r", returns="i") - def opimpl_ptr_ne(self, a, b): + def opimpl_ptr_ne(a, b): return int(a != b) @arguments("r", returns="i") - def opimpl_ptr_iszero(self, a): + def opimpl_ptr_iszero(a): return int(not a) @arguments("r", returns="i") - def opimpl_ptr_nonzero(self, a): + def opimpl_ptr_nonzero(a): return int(bool(a)) - @arguments("i") + @arguments("self", "i") def opimpl_int_return(self, a): self.result_i = a raise LeaveFrame @arguments("i", returns="i") - def opimpl_int_copy(self, a): + def opimpl_int_copy(a): return a @arguments("r", returns="r") - def opimpl_ref_copy(self, a): + def opimpl_ref_copy(a): return a @arguments("f", returns="f") - def opimpl_float_copy(self, a): + def opimpl_float_copy(a): return a opimpl_int_guard_value = opimpl_int_copy @@ -375,95 +377,95 @@ opimpl_float_guard_value = opimpl_float_copy @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_lt(self, target, a, b, pc): + def opimpl_goto_if_not_int_lt(target, a, b, pc): if a < b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_le(self, target, a, b, pc): + def opimpl_goto_if_not_int_le(target, a, b, pc): if a <= b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_eq(self, target, a, b, pc): + def opimpl_goto_if_not_int_eq(target, a, b, pc): if a == b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_ne(self, target, a, b, pc): + def opimpl_goto_if_not_int_ne(target, a, b, pc): if a != b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_gt(self, target, a, b, pc): + def opimpl_goto_if_not_int_gt(target, a, b, pc): if a > b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_ge(self, target, a, b, pc): + def opimpl_goto_if_not_int_ge(target, a, b, pc): if a >= b: return pc else: return target @arguments("L", "i", "pc", returns="L") - def opimpl_goto_if_not_int_is_zero(self, target, a, pc): + def opimpl_goto_if_not_int_is_zero(target, a, pc): if not a: return pc else: return target @arguments("L", "i", "pc", returns="L") - def opimpl_goto_if_not_int_is_true(self, target, a, pc): + def opimpl_goto_if_not_int_is_true(target, a, pc): if a: return pc else: return target @arguments("L", "r", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_eq(self, target, a, b, pc): + def opimpl_goto_if_not_ptr_eq(target, a, b, pc): if a == b: return pc else: return target @arguments("L", "r", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_ne(self, target, a, b, pc): + def opimpl_goto_if_not_ptr_ne(target, a, b, pc): if a != b: return pc else: return target @arguments("L", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_iszero(self, target, a, pc): + def opimpl_goto_if_not_ptr_iszero(target, a, pc): if not a: return pc else: return target @arguments("L", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_nonzero(self, target, a, pc): + def opimpl_goto_if_not_ptr_nonzero(target, a, pc): if a: return pc else: return target @arguments("L", returns="L") - def opimpl_goto(self, target): + def opimpl_goto(target): return target @arguments("i", "d", "pc", returns="L") - def opimpl_switch(self, switchvalue, switchdict, pc): + def opimpl_switch(switchvalue, switchdict, pc): assert isinstance(switchdict, SwitchDictDescr) try: return switchdict.dict[switchvalue] @@ -471,13 +473,13 @@ return pc @arguments("L") - def opimpl_catch_exception(self, target): + def opimpl_catch_exception(target): """This is a no-op when run normally. When an exception occurs and the instruction that raised is immediately followed by a catch_exception, then the code in handle_exception_in_frame() will capture the exception and jump to 'target'.""" - @arguments("i", "L", "pc", returns="L") + @arguments("self", "i", "L", "pc", returns="L") def opimpl_goto_if_exception_mismatch(self, vtable, target, pc): adr = llmemory.cast_int_to_adr(vtable) bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) @@ -488,20 +490,20 @@ else: return target - @arguments(returns="i") + @arguments("self", returns="i") def opimpl_last_exception(self): real_instance = self.exception_last_value assert real_instance adr = llmemory.cast_ptr_to_adr(real_instance.typeptr) return llmemory.cast_adr_to_int(adr) - @arguments(returns="r") + @arguments("self", returns="r") def opimpl_last_exc_value(self): real_instance = self.exception_last_value assert real_instance return lltype.cast_opaque_ptr(llmemory.GCREF, real_instance) - @arguments() + @arguments("self") def opimpl_reraise(self): real_instance = self.exception_last_value assert real_instance @@ -510,65 +512,65 @@ # ---------- # the following operations are directly implemented by the backend - @arguments("i", "d", "R", returns="i") + @arguments("self", "i", "d", "R", returns="i") def opimpl_residual_call_r_i(self, func, calldescr, args_r): return self.cpu.bh_call_i(func, calldescr, None, args_r, None) - @arguments("i", "d", "R", returns="r") + @arguments("self", "i", "d", "R", returns="r") def opimpl_residual_call_r_r(self, func, calldescr, args_r): return self.cpu.bh_call_r(func, calldescr, None, args_r, None) - @arguments("i", "d", "R", returns="f") + @arguments("self", "i", "d", "R", returns="f") def opimpl_residual_call_r_f(self, func, calldescr, args_r): return self.cpu.bh_call_f(func, calldescr, None, args_r, None) - @arguments("i", "d", "R") + @arguments("self", "i", "d", "R") def opimpl_residual_call_r_v(self, func, calldescr, args_r): self.cpu.bh_call_v(func, calldescr, None, args_r, None) - @arguments("i", "d", "I", "R", returns="i") + @arguments("self", "i", "d", "I", "R", returns="i") def opimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_i(func, calldescr, args_i, args_r, None) - @arguments("i", "d", "I", "R", returns="r") + @arguments("self", "i", "d", "I", "R", returns="r") def opimpl_residual_call_ir_r(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_r(func, calldescr, args_i, args_r, None) - @arguments("i", "d", "I", "R", returns="f") + @arguments("self", "i", "d", "I", "R", returns="f") def opimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, None) - @arguments("i", "d", "I", "R") + @arguments("self", "i", "d", "I", "R") def opimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) - @arguments("i", "d", "I", "R", "F", returns="i") + @arguments("self", "i", "d", "I", "R", "F", returns="i") def opimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_i(func, calldescr, args_i, args_r, args_f) - @arguments("i", "d", "I", "R", "F", returns="r") + @arguments("self", "i", "d", "I", "R", "F", returns="r") def opimpl_residual_call_irf_r(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_r(func, calldescr, args_i, args_r, args_f) - @arguments("i", "d", "I", "R", "F", returns="f") + @arguments("self", "i", "d", "I", "R", "F", returns="f") def opimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) - @arguments("i", "d", "I", "R", "F") + @arguments("self", "i", "d", "I", "R", "F") def opimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) - @arguments("d", "i", returns="r") + @arguments("self", "d", "i", returns="r") def opimpl_new_array(self, arraydescr, length): return self.cpu.bh_new_array(arraydescr, length) - @arguments("d", "r", "i", "r") + @arguments("self", "d", "r", "i", "r") def opimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) - @arguments("r", "d", returns="i") + @arguments("self", "r", "d", returns="i") def opimpl_getfield_gc_i(self, struct, fielddescr): return self.cpu.bh_getfield_gc_i(struct, fielddescr) - @arguments("r", "d", returns="i") + @arguments("self", "r", "d", returns="i") def opimpl_getfield_gc_c(self, struct, fielddescr): return self.cpu.bh_getfield_gc_c(struct, fielddescr) - @arguments("r", "d", returns="i") + @arguments("self", "r", "d", returns="i") def opimpl_getfield_gc_u(self, struct, fielddescr): return self.cpu.bh_getfield_gc_u(struct, fielddescr) - @arguments("r", "d", returns="r") + @arguments("self", "r", "d", returns="r") def opimpl_getfield_gc_r(self, struct, fielddescr): return self.cpu.bh_getfield_gc_r(struct, fielddescr) - @arguments("r", "d", returns="f") + @arguments("self", "r", "d", returns="f") def opimpl_getfield_gc_f(self, struct, fielddescr): return self.cpu.bh_getfield_gc_f(struct, fielddescr) @@ -578,19 +580,19 @@ opimpl_getfield_gc_r_pure = opimpl_getfield_gc_r opimpl_getfield_gc_f_pure = opimpl_getfield_gc_f - @arguments("i", "d", returns="i") + @arguments("self", "i", "d", returns="i") def opimpl_getfield_raw_i(self, struct, fielddescr): return self.cpu.bh_getfield_raw_i(struct, fielddescr) - @arguments("i", "d", returns="i") + @arguments("self", "i", "d", returns="i") def opimpl_getfield_raw_c(self, struct, fielddescr): return self.cpu.bh_getfield_raw_c(struct, fielddescr) - @arguments("i", "d", returns="i") + @arguments("self", "i", "d", returns="i") def opimpl_getfield_raw_u(self, struct, fielddescr): return self.cpu.bh_getfield_raw_u(struct, fielddescr) - @arguments("i", "d", returns="r") + @arguments("self", "i", "d", returns="r") def opimpl_getfield_raw_r(self, struct, fielddescr): return self.cpu.bh_getfield_raw_r(struct, fielddescr) - @arguments("i", "d", returns="f") + @arguments("self", "i", "d", returns="f") def opimpl_getfield_raw_f(self, struct, fielddescr): return self.cpu.bh_getfield_raw_f(struct, fielddescr) @@ -600,50 +602,50 @@ opimpl_getfield_raw_r_pure = opimpl_getfield_raw_r opimpl_getfield_raw_f_pure = opimpl_getfield_raw_f - @arguments("r", "d", "i") + @arguments("self", "r", "d", "i") def opimpl_setfield_gc_i(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) - @arguments("r", "d", "i") + @arguments("self", "r", "d", "i") def opimpl_setfield_gc_c(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) - @arguments("r", "d", "i") + @arguments("self", "r", "d", "i") def opimpl_setfield_gc_u(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) - @arguments("r", "d", "r") + @arguments("self", "r", "d", "r") def opimpl_setfield_gc_r(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) - @arguments("r", "d", "f") + @arguments("self", "r", "d", "f") def opimpl_setfield_gc_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) - @arguments("i", "d", "i") + @arguments("self", "i", "d", "i") def opimpl_setfield_raw_i(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) - @arguments("i", "d", "i") + @arguments("self", "i", "d", "i") def opimpl_setfield_raw_c(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) - @arguments("i", "d", "i") + @arguments("self", "i", "d", "i") def opimpl_setfield_raw_u(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) - @arguments("i", "d", "r") + @arguments("self", "i", "d", "r") def opimpl_setfield_raw_r(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) - @arguments("i", "d", "f") + @arguments("self", "i", "d", "f") def opimpl_setfield_raw_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) - @arguments("d", returns="r") + @arguments("self", "d", returns="r") def opimpl_new(self, descr): return self.cpu.bh_new(descr) - @arguments("d", returns="r") + @arguments("self", "d", returns="r") def opimpl_new_with_vtable(self, descr): return self.cpu.bh_new_with_vtable(descr) - @arguments("r", returns="i") + @arguments("self", "r", returns="i") def opimpl_classof(self, struct): return self.cpu.bh_classof(struct) - @arguments("r", returns="i") + @arguments("self", "r", returns="i") def opimpl_cast_ptr_to_int(self, p): return self.cpu.bh_cast_ptr_to_int(p) From afa at codespeak.net Wed Apr 28 14:56:54 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 14:56:54 +0200 (CEST) Subject: [pypy-svn] r74172 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428125654.266CA282B9D@codespeak.net> Author: afa Date: Wed Apr 28 14:56:52 2010 New Revision: 74172 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_GetAttr, PyObject_DelAttr Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 14:56:52 2010 @@ -71,6 +71,13 @@ def PyObject_Not(space, w_obj): return not space.is_true(w_obj) + at cpython_api([PyObject, PyObject], PyObject) +def PyObject_GetAttr(space, w_obj, w_name): + """Retrieve an attribute named attr_name from object o. Returns the attribute + value on success, or NULL on failure. This is the equivalent of the Python + expression o.attr_name.""" + return space.getattr(w_obj, w_name) + @cpython_api([PyObject, CONST_STRING], PyObject) def PyObject_GetAttrString(space, w_obj, name_ptr): """Retrieve an attribute named attr_name from object o. Returns the attribute @@ -107,6 +114,13 @@ operation.setattr(space, w_obj, w_name, w_value) return 0 + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_DelAttr(space, w_obj, w_name): + """Delete attribute named attr_name, for object o. Returns -1 on failure. + This is the equivalent of the Python statement del o.attr_name.""" + space.delattr(w_obj, w_name) + return 0 + @cpython_api([PyObject], lltype.Void) def PyObject_ClearWeakRefs(space, w_object): w_object.clear_all_weakrefs() Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 14:56:52 2010 @@ -3023,19 +3023,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyObject_GetAttr(space, o, attr_name): - """Retrieve an attribute named attr_name from object o. Returns the attribute - value on success, or NULL on failure. This is the equivalent of the Python - expression o.attr_name.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyObject_DelAttr(space, o, attr_name): - """Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement del o.attr_name.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyObject_DelAttrString(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 28 14:56:52 2010 @@ -54,7 +54,7 @@ rffi.free_charp(buf) assert x.test == 20 - def test_getattr_string(self, space, api): + def test_getattr(self, space, api): charp1 = rffi.str2charp("__len__") charp2 = rffi.str2charp("not_real") assert api.PyObject_GetAttrString(space.wrap(""), charp1) @@ -64,6 +64,10 @@ rffi.free_charp(charp1) rffi.free_charp(charp2) + assert api.PyObject_GetAttr(space.wrap(""), space.wrap("__len__")) + assert api.PyObject_DelAttr(space.wrap(""), space.wrap("__len__")) == -1 + api.PyErr_Clear() + def test_getitem(self, space, api): w_t = space.wrap((1, 2, 3, 4, 5)) assert space.unwrap(api.PyObject_GetItem(w_t, space.wrap(3))) == 4 From afa at codespeak.net Wed Apr 28 14:57:26 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 14:57:26 +0200 (CEST) Subject: [pypy-svn] r74173 - in pypy/branch/cpython-extension/pypy/module/cpyext: . src Message-ID: <20100428125726.BED3B282B9D@codespeak.net> Author: afa Date: Wed Apr 28 14:57:25 2010 New Revision: 74173 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c Log: PyObject_CallFunctionObjArgs, PyObject_CallMethodObjArgs Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 28 14:57:25 2010 @@ -254,7 +254,7 @@ 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', - 'PyObject_CallMethod', + 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', 'PyBuffer_FromMemory', 'PyBuffer_New', 'PyBuffer_Type', 'init_bufferobject', '_PyArg_NoKeywords', 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', Modified: pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/modsupport.c Wed Apr 28 14:57:25 2010 @@ -592,6 +592,79 @@ return retval; } +static PyObject * +objargs_mktuple(va_list va) +{ + int i, n = 0; + va_list countva; + PyObject *result, *tmp; + +#ifdef VA_LIST_IS_ARRAY + memcpy(countva, va, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(countva, va); +#else + countva = va; +#endif +#endif + + while (((PyObject *)va_arg(countva, PyObject *)) != NULL) + ++n; + result = PyTuple_New(n); + if (result != NULL && n > 0) { + for (i = 0; i < n; ++i) { + tmp = (PyObject *)va_arg(va, PyObject *); + PyTuple_SET_ITEM(result, i, tmp); + Py_INCREF(tmp); + } + } + return result; +} + +PyObject * +PyObject_CallFunctionObjArgs(PyObject *callable, ...) +{ + PyObject *args, *tmp; + va_list vargs; + + /* count the args */ + va_start(vargs, callable); + args = objargs_mktuple(vargs); + va_end(vargs); + if (args == NULL) + return NULL; + tmp = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + return tmp; +} + +PyObject * +PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...) +{ + PyObject *args, *tmp; + va_list vargs; + + callable = PyObject_GetAttr(callable, name); + if (callable == NULL) + return NULL; + + /* count the args */ + va_start(vargs, name); + args = objargs_mktuple(vargs); + va_end(vargs); + if (args == NULL) { + Py_DECREF(callable); + return NULL; + } + tmp = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + Py_DECREF(callable); + + return tmp; +} + int PyModule_AddObject(PyObject *m, const char *name, PyObject *o) { From afa at codespeak.net Wed Apr 28 15:04:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 15:04:48 +0200 (CEST) Subject: [pypy-svn] r74174 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428130448.E3B22282B9D@codespeak.net> Author: afa Date: Wed Apr 28 15:04:47 2010 New Revision: 74174 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Log: PyDict_GetItem never raises: it returns NULL when the key does not exists *and* on any other error. + add PyDict_Clear Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Wed Apr 28 15:04:47 2010 @@ -11,12 +11,14 @@ PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") - at cpython_api([PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject], PyObject, borrowed=True, error=CANNOT_FAIL) def PyDict_GetItem(space, w_dict, w_key): - if PyDict_Check(space, w_dict): - return space.getitem(w_dict, w_key) - else: - PyErr_BadInternalCall(space) + try: + w_res = space.getitem(w_dict, w_key) + except: + return None + register_container(space, w_dict) + return w_res @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -45,15 +47,16 @@ else: PyErr_BadInternalCall(space) - at cpython_api([PyObject, CONST_STRING], PyObject, borrowed=True) + at cpython_api([PyObject, CONST_STRING], PyObject, borrowed=True, error=CANNOT_FAIL) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" - if not PyDict_Check(space, w_dict): - PyErr_BadInternalCall(space) - w_res = space.finditem_str(w_dict, rffi.charp2str(key)) + try: + w_res = space.finditem_str(w_dict, rffi.charp2str(key)) + except: + w_res = None if w_res is None: - raise OperationError(space.w_KeyError, space.wrap("Key not found")) + return None register_container(space, w_dict) return w_res @@ -64,6 +67,11 @@ len(p) on a dictionary.""" return space.int_w(space.len(w_obj)) + at cpython_api([PyObject], lltype.Void) +def PyDict_Clear(space, w_obj): + """Empty an existing dictionary of all key-value pairs.""" + space.call_method(w_obj, "clear") + @cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) def PyDict_Next(space, w_obj, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Wed Apr 28 15:04:47 2010 @@ -1,3 +1,4 @@ +from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest class TestDictObject(BaseApiTest): @@ -19,8 +20,12 @@ space.delitem(d, space.wrap("name")) assert not api.PyDict_GetItem(d, space.wrap("name")) - assert api.PyErr_Occurred() is space.w_KeyError - api.PyErr_Clear() + assert not api.PyErr_Occurred() + + buf = rffi.str2charp("name") + assert not api.PyDict_GetItemString(d, buf) + rffi.free_charp(buf) + assert not api.PyErr_Occurred() assert api.PyDict_DelItem(d, space.wrap("c")) == 0 assert api.PyDict_DelItem(d, space.wrap("name")) < 0 @@ -28,6 +33,10 @@ api.PyErr_Clear() assert api.PyDict_Size(d) == 0 + d = space.wrap({'a': 'b'}) + api.PyDict_Clear(d) + assert api.PyDict_Size(d) == 0 + def test_check(self, space, api): d = api.PyDict_New() assert api.PyDict_Check(d) From afa at codespeak.net Wed Apr 28 15:27:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 15:27:11 +0200 (CEST) Subject: [pypy-svn] r74175 - pypy/branch/cpython-extension/pypy/module/cpyext/include Message-ID: <20100428132711.8C8CE282B9D@codespeak.net> Author: afa Date: Wed Apr 28 15:27:10 2010 New Revision: 74175 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h Log: Expose recently exported functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/eval.h Wed Apr 28 15:27:10 2010 @@ -16,6 +16,8 @@ PyObject * PyEval_CallMethod(PyObject *obj, const char *name, const char *format, ...); PyObject * PyObject_CallFunction(PyObject *obj, char *format, ...); PyObject * PyObject_CallMethod(PyObject *obj, char *name, char *format, ...); +PyObject * PyObject_CallFunctionObjArgs(PyObject *callable, ...); +PyObject * PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...); /* These constants are also defined in cpyext/eval.py */ #define Py_single_input 256 From afa at codespeak.net Wed Apr 28 15:39:37 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 15:39:37 +0200 (CEST) Subject: [pypy-svn] r74176 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428133937.E16C3282B9E@codespeak.net> Author: afa Date: Wed Apr 28 15:39:35 2010 New Revision: 74176 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: PyCObject_Import Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Wed Apr 28 15:39:35 2010 @@ -2,8 +2,10 @@ from pypy.interpreter.typedef import TypeDef from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.pyobject import make_ref, make_typedescr +from pypy.module.cpyext.import_ import PyImport_ImportModule +from pypy.module.cpyext.object import PyObject_GetAttrString from pypy.module.cpyext.api import generic_cpy_call, cpython_api, bootstrap_function, \ - PyObject, cpython_struct, PyObjectFields, build_type_checkers + PyObject, cpython_struct, PyObjectFields, build_type_checkers, CONST_STRING destructor_short = lltype.Ptr(lltype.FuncType([rffi.VOIDP_real], lltype.Void)) @@ -83,3 +85,12 @@ created with.""" assert isinstance(w_obj, W_PyCObjectFromVoidPtr) return w_obj.pyo.c_cobject + + at cpython_api([CONST_STRING, CONST_STRING], rffi.VOIDP_real, + error=lltype.nullptr(rffi.VOIDP_real.TO)) +def PyCObject_Import(space, module_name, name): + """Return the object void * that the PyCObject self was + created with.""" + w_mod = PyImport_ImportModule(space, module_name) + w_obj = PyObject_GetAttrString(space, w_mod, name) + return PyCObject_AsVoidPtr(space, w_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 15:39:35 2010 @@ -1464,21 +1464,6 @@ only be called from the main thread.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, PyObject, PyObject], PyObject) -def PyErr_NewException(space, name, base, dict): - """This utility function creates and returns a new exception object. The name - argument must be the name of the new exception, a C string of the form - module.class. The base and dict arguments are normally NULL. This - creates a class object derived from Exception (accessible in C as - PyExc_Exception). - - The __module__ attribute of the new class is set to the first part (up - to the last dot) of the name argument, and the class name is set to the last - part (after the last dot). The base argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The dict - argument can be used to specify a dictionary of class variables and methods.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARP, PyObject, PyObject], PyObject) def PyErr_NewExceptionWithDoc(space, name, doc, base, dict): """Same as PyErr_NewException(), except that the new exception class can Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Wed Apr 28 15:39:35 2010 @@ -16,3 +16,17 @@ obj = api.PyCObject_FromVoidPtrAndDesc(ptr, ptr, lltype.nullptr(destructor_short.TO)) api.Py_DecRef(obj) + + def test_pycobject_import(self, space, api): + ptr = rffi.cast(rffi.VOIDP_real, 1234) + obj = api.PyCObject_FromVoidPtr(ptr, lltype.nullptr(destructor_short.TO)) + space.setattr(space.sys, space.wrap("_cpyext_cobject"), obj) + + charp1 = rffi.str2charp("sys") + charp2 = rffi.str2charp("_cpyext_cobject") + assert api.PyCObject_Import(charp1, charp2) == ptr + rffi.free_charp(charp1) + rffi.free_charp(charp2) + + api.Py_DecRef(obj) + space.delattr(space.sys, space.wrap("_cpyext_cobject")) From agaynor at codespeak.net Wed Apr 28 16:04:21 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 28 Apr 2010 16:04:21 +0200 (CEST) Subject: [pypy-svn] r74177 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428140421.4E82D282B9E@codespeak.net> Author: agaynor Date: Wed Apr 28 16:04:19 2010 New Revision: 74177 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: Implement PyObject_Hash. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 16:04:19 2010 @@ -299,3 +299,10 @@ return rffi.cast(rffi.INT_real, fd) + + at cpython_api([PyObject], lltype.Signed, error=-1) +def PyObject_Hash(space, w_obj): + """ + Compute and return the hash value of an object o. On failure, return -1. + This is the equivalent of the Python expression hash(o).""" + return space.int_w(space.hash(w_obj)) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 16:04:19 2010 @@ -3101,16 +3101,6 @@ raise NotImplementedError @cpython_api([PyObject], lltype.Signed, error=-1) -def PyObject_Hash(space, o): - """ - - - - Compute and return the hash value of an object o. On failure, return -1. - This is the equivalent of the Python expression hash(o).""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Signed, error=-1) def PyObject_HashNotImplemented(space, o): """Set a TypeError indicating that type(o) is not hashable and return -1. This function receives special treatment when stored in a tp_hash slot, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 28 16:04:19 2010 @@ -157,3 +157,10 @@ return File""") w_f = space.call_function(w_File) assert api.PyObject_AsFileDescriptor(w_f) == 42 + + def test_hash(self, space, api): + assert api.PyObject_Hash(space.wrap(72)) == 72 + assert api.PyObject_Hash(space.wrap(-1)) == -1 + assert (api.PyObject_Hash(space.wrap([])) == -1 and + api.PyErr_Occurred() is space.w_TypeError) + api.PyErr_Clear() From afa at codespeak.net Wed Apr 28 16:34:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 16:34:15 +0200 (CEST) Subject: [pypy-svn] r74178 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100428143415.D16D0282B9D@codespeak.net> Author: afa Date: Wed Apr 28 16:34:13 2010 New Revision: 74178 Added: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Log: Expose PyFunctionObject and its "func_name" field Add PyMethod_Function, PyMethod_Class Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Wed Apr 28 16:34:13 2010 @@ -67,6 +67,7 @@ import pypy.module.cpyext.datetime import pypy.module.cpyext.complexobject import pypy.module.cpyext.weakrefobject +import pypy.module.cpyext.funcobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Wed Apr 28 16:34:13 2010 @@ -0,0 +1,52 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.api import ( + PyObjectFields, generic_cpy_call, + cpython_api, bootstrap_function, cpython_struct, build_type_checkers) +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr +from pypy.interpreter.function import Function, Method + +PyFunctionObjectStruct = lltype.ForwardReference() +PyFunctionObject = lltype.Ptr(PyFunctionObjectStruct) +PyFunctionObjectFields = PyObjectFields + \ + (("func_name", PyObject),) +cpython_struct("PyFunctionObject", PyFunctionObjectFields, PyFunctionObjectStruct) + + at bootstrap_function +def init_functionobject(space): + make_typedescr(Function.typedef, + basestruct=PyFunctionObject.TO, + attach=function_attach, + dealloc=function_dealloc) + +PyFunction_Check, PyFunction_CheckExact = build_type_checkers("Function", Function) +PyMethod_Check, PyMethod_CheckExact = build_type_checkers("Method", Method) + +def function_attach(space, py_obj, w_obj): + py_func = rffi.cast(PyFunctionObject, py_obj) + assert isinstance(w_obj, Function) + py_func.c_func_name = make_ref(space, space.wrap(w_obj.name)) + + at cpython_api([PyObject], lltype.Void, external=False) +def function_dealloc(space, py_obj): + py_func = rffi.cast(PyFunctionObject, py_obj) + Py_DecRef(space, py_func.c_func_name) + # standard dealloc + pto = py_obj.c_ob_type + obj_voidp = rffi.cast(rffi.VOIDP_real, py_obj) + generic_cpy_call(space, pto.c_tp_free, obj_voidp) + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) + + at cpython_api([PyObject], PyObject, borrowed=True) +def PyMethod_Function(space, w_method): + """Return the function object associated with the method meth.""" + assert isinstance(w_method, Method) + return w_method.w_function + + at cpython_api([PyObject], PyObject, borrowed=True) +def PyMethod_Class(space, w_method): + """Return the class object from which the method meth was created; if this was + created from an instance, it will be the class of the instance.""" + assert isinstance(w_method, Method) + return w_method.w_class + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/Python.h Wed Apr 28 16:34:13 2010 @@ -102,6 +102,7 @@ #include "boolobject.h" #include "floatobject.h" #include "methodobject.h" +#include "funcobject.h" #include "modsupport.h" #include "pythonrun.h" Added: pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h Wed Apr 28 16:34:13 2010 @@ -0,0 +1,21 @@ + +/* Function object interface */ + +#ifndef Py_FUNCOBJECT_H +#define Py_FUNCOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + PyObject_HEAD + PyObject *func_name; /* The __name__ attribute, a string object */ +} PyFunctionObject; + +#define PyMethod_GET_CLASS(obj) PyMethod_Class(obj) +#define PyMethod_GET_FUNCTION(obj) PyMethod_Function(obj) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_FUNCOBJECT_H */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Wed Apr 28 16:34:13 2010 @@ -89,7 +89,6 @@ def descr_method_repr(self): return self.getrepr(self.space, "built-in method '%s' of '%s' object" % (self.name, self.w_objclass.getname(self.space, '?'))) -PyMethod_Check, PyMethod_CheckExact = build_type_checkers("Method", W_PyCMethodObject) PyCFunction_Check, PyCFunction_CheckExact = build_type_checkers("CFunction", W_PyCFunctionObject) class W_PyCWrapperObject(Wrappable): Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py Wed Apr 28 16:34:13 2010 @@ -0,0 +1,29 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.funcobject import PyFunctionObject +from pypy.interpreter.function import Function, Method + +class TestFunctionObject(BaseApiTest): + def test_function(self, space, api): + w_function = space.appexec([], """(): + def f(): pass + return f + """) + ref = make_ref(space, w_function) + assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is + space.gettypeobject(Function.typedef)) + assert "f" == space.unwrap( + from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name)) + api.Py_DecRef(ref) + + def test_method(self, space, api): + w_method = space.appexec([], """(): + class C(list): + def method(self): pass + return C().method + """) + + w_function = space.getattr(w_method, space.wrap("im_func")) + assert space.is_w(api.PyMethod_Function(w_method), w_function) From afa at codespeak.net Wed Apr 28 16:44:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 16:44:46 +0200 (CEST) Subject: [pypy-svn] r74179 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428144446.012F8282B9D@codespeak.net> Author: afa Date: Wed Apr 28 16:44:45 2010 New Revision: 74179 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_Type Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 16:44:45 2010 @@ -166,6 +166,17 @@ return from_ref(space, rffi.cast(PyObject, op)) # XXX likewise @cpython_api([PyObject], PyObject) +def PyObject_Type(space, w_obj): + """When o is non-NULL, returns a type object corresponding to the object type + of object o. On failure, raises SystemError and returns NULL. This + is equivalent to the Python expression type(o). This function increments the + reference count of the return value. There's really no reason to use this + function instead of the common expression o->ob_type, which returns a + pointer of type PyTypeObject*, except when the incremented reference + count is needed.""" + return space.type(w_obj) + + at cpython_api([PyObject], PyObject) def PyObject_Str(space, w_obj): return space.str(w_obj) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 16:44:45 2010 @@ -2836,27 +2836,6 @@ raise NotImplementedError @cpython_api([PyObject], PyObject, borrowed=True) -def PyMethod_Class(space, meth): - """Return the class object from which the method meth was created; if this was - created from an instance, it will be the class of the instance.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject, borrowed=True) -def PyMethod_GET_CLASS(space, meth): - """Macro version of PyMethod_Class() which avoids error checking.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject, borrowed=True) -def PyMethod_Function(space, meth): - """Return the function object associated with the method meth.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject, borrowed=True) -def PyMethod_GET_FUNCTION(space, meth): - """Macro version of PyMethod_Function() which avoids error checking.""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject, borrowed=True) def PyMethod_Self(space, meth): """Return the instance associated with the method meth if it is bound, otherwise return NULL.""" @@ -3109,21 +3088,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_Type(space, o): - """ - - - - When o is non-NULL, returns a type object corresponding to the object type - of object o. On failure, raises SystemError and returns NULL. This - is equivalent to the Python expression type(o). This function increments the - reference count of the return value. There's really no reason to use this - function instead of the common expression o->ob_type, which returns a - pointer of type PyTypeObject*, except when the incremented reference - count is needed.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_SetItem(space, o, key, v): """Map the object key to the value v. Returns -1 on failure. This is the Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 28 16:44:45 2010 @@ -164,3 +164,6 @@ assert (api.PyObject_Hash(space.wrap([])) == -1 and api.PyErr_Occurred() is space.w_TypeError) api.PyErr_Clear() + + def test_type(self, space, api): + assert api.PyObject_Type(space.wrap(72)) is space.w_int From afa at codespeak.net Wed Apr 28 16:47:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 16:47:00 +0200 (CEST) Subject: [pypy-svn] r74180 - in pypy/branch/cpython-extension/pypy: module/cpyext/include module/cpyext/src module/cpyext/test tool Message-ID: <20100428144700.3DCF2282B9D@codespeak.net> Author: afa Date: Wed Apr 28 16:46:58 2010 New Revision: 74180 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/ (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/bufferobject.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/pyconfig.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/pyport.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/pythread.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/sliceobject.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/include/warnings.h (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/src/ (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/src/bufferobject.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/array.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/dotted.c (props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py (contents, props changed) pypy/branch/cpython-extension/pypy/tool/fixeol Log: Add .h and .c files to fixeol Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/funcobject.h Wed Apr 28 16:46:58 2010 @@ -1,21 +1,21 @@ - -/* Function object interface */ - -#ifndef Py_FUNCOBJECT_H -#define Py_FUNCOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - PyObject_HEAD - PyObject *func_name; /* The __name__ attribute, a string object */ -} PyFunctionObject; - -#define PyMethod_GET_CLASS(obj) PyMethod_Class(obj) -#define PyMethod_GET_FUNCTION(obj) PyMethod_Function(obj) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_FUNCOBJECT_H */ + +/* Function object interface */ + +#ifndef Py_FUNCOBJECT_H +#define Py_FUNCOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + PyObject_HEAD + PyObject *func_name; /* The __name__ attribute, a string object */ +} PyFunctionObject; + +#define PyMethod_GET_CLASS(obj) PyMethod_Class(obj) +#define PyMethod_GET_FUNCTION(obj) PyMethod_Function(obj) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_FUNCOBJECT_H */ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py Wed Apr 28 16:46:58 2010 @@ -1,29 +1,29 @@ -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref -from pypy.module.cpyext.funcobject import PyFunctionObject -from pypy.interpreter.function import Function, Method - -class TestFunctionObject(BaseApiTest): - def test_function(self, space, api): - w_function = space.appexec([], """(): - def f(): pass - return f - """) - ref = make_ref(space, w_function) - assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is - space.gettypeobject(Function.typedef)) - assert "f" == space.unwrap( - from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name)) - api.Py_DecRef(ref) - - def test_method(self, space, api): - w_method = space.appexec([], """(): - class C(list): - def method(self): pass - return C().method - """) - - w_function = space.getattr(w_method, space.wrap("im_func")) - assert space.is_w(api.PyMethod_Function(w_method), w_function) +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.funcobject import PyFunctionObject +from pypy.interpreter.function import Function, Method + +class TestFunctionObject(BaseApiTest): + def test_function(self, space, api): + w_function = space.appexec([], """(): + def f(): pass + return f + """) + ref = make_ref(space, w_function) + assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is + space.gettypeobject(Function.typedef)) + assert "f" == space.unwrap( + from_ref(space, rffi.cast(PyFunctionObject, ref).c_func_name)) + api.Py_DecRef(ref) + + def test_method(self, space, api): + w_method = space.appexec([], """(): + class C(list): + def method(self): pass + return C().method + """) + + w_function = space.getattr(w_method, space.wrap("im_func")) + assert space.is_w(api.PyMethod_Function(w_method), w_function) Modified: pypy/branch/cpython-extension/pypy/tool/fixeol ============================================================================== --- pypy/branch/cpython-extension/pypy/tool/fixeol (original) +++ pypy/branch/cpython-extension/pypy/tool/fixeol Wed Apr 28 16:46:58 2010 @@ -48,7 +48,7 @@ return True def checkeolfile(path): - return path.ext in ('.txt', '.py', '.asc') + return path.ext in ('.txt', '.py', '.asc', '.h', '.c') def fixdirectory(path): print "+ checking directory", path, From afa at codespeak.net Wed Apr 28 17:18:57 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 17:18:57 +0200 (CEST) Subject: [pypy-svn] r74181 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428151857.3857D282B9D@codespeak.net> Author: afa Date: Wed Apr 28 17:18:55 2010 New Revision: 74181 Added: pypy/branch/cpython-extension/pypy/module/cpyext/classobject.py (contents, props changed) pypy/branch/cpython-extension/pypy/module/cpyext/test/test_classobject.py (contents, props changed) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: old-style classes: PyClass_Check, PyInstance_Check, PyInstance_NewRaw, _PyInstance_Lookup Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Wed Apr 28 17:18:55 2010 @@ -68,6 +68,7 @@ import pypy.module.cpyext.complexobject import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject +import pypy.module.cpyext.classobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/branch/cpython-extension/pypy/module/cpyext/classobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/classobject.py Wed Apr 28 17:18:55 2010 @@ -0,0 +1,30 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.cpyext.api import ( + PyObjectFields, CANNOT_FAIL, + cpython_api, bootstrap_function, cpython_struct, build_type_checkers) +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr +from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.module.__builtin__.interp_classobj import W_ClassObject, W_InstanceObject + +PyClass_Check, PyClass_CheckExact = build_type_checkers("Class", W_ClassObject) +PyInstance_Check, PyInstance_CheckExact = build_type_checkers("Instance", W_InstanceObject) + + at cpython_api([PyObject, PyObject], PyObject) +def PyInstance_NewRaw(space, w_class, w_dict): + """Create a new instance of a specific class without calling its constructor. + class is the class of new object. The dict parameter will be used as the + object's __dict__; if NULL, a new dictionary will be created for the + instance.""" + if not PyClass_Check(space, w_class): + return PyErr_BadInternalCall(space) + return W_InstanceObject(space, w_class, w_dict) + + at cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) +def _PyInstance_Lookup(space, w_instance, w_name): + assert isinstance(w_instance, W_InstanceObject) + w_result = space.finditem(w_instance.w_dict, w_name) + if w_result is not None: + return w_result + return w_instance.w_class.lookup(space, w_name) + + Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 17:18:55 2010 @@ -665,36 +665,17 @@ be a cell object.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyClass_Check(space, o): - """Return true if the object o is a class object, including instances of types - derived from the standard class object. Return false in all other cases.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyClass_IsSubclass(space, klass, base): """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyInstance_Check(space, obj): - """Return true if obj is an instance.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyInstance_New(space, cls, arg, kw): """Create a new instance of a specific class. The parameters arg and kw are used as the positional and keyword parameters to the object's constructor.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], PyObject) -def PyInstance_NewRaw(space, cls, dict): - """Create a new instance of a specific class without calling its constructor. - class is the class of new object. The dict parameter will be used as the - object's __dict__; if NULL, a new dictionary will be created for the - instance.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.VOIDP_real, error=lltype.nullptr(rffi.VOIDP.TO)) def PyCObject_GetDesc(space, self): """Return the description void * that the PyCObject self was Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_classobject.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_classobject.py Wed Apr 28 17:18:55 2010 @@ -0,0 +1,42 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.interpreter.function import Function, Method + +class TestClassObject(BaseApiTest): + def test_newinstance(self, space, api): + w_class = space.appexec([], """(): + class C: + x = None + def __init__(self): + self.x = 1 + return C + """) + + assert api.PyClass_Check(w_class) + + w_instance = api.PyInstance_NewRaw(w_class, None) + assert api.PyInstance_Check(w_instance) + assert space.getattr(w_instance, space.wrap('x')) is space.w_None + + w_instance = api.PyInstance_NewRaw(w_class, space.wrap(dict(a=3))) + assert space.getattr(w_instance, space.wrap('x')) is space.w_None + assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + + def test_lookup(self, space, api): + w_instance = space.appexec([], """(): + class C: + def __init__(self): + self.x = None + def f(self): pass + return C() + """) + + assert api.PyInstance_Check(w_instance) + assert api.PyObject_GetAttr(w_instance, space.wrap('x')) is space.w_None + assert api._PyInstance_Lookup(w_instance, space.wrap('x')) is space.w_None + assert api._PyInstance_Lookup(w_instance, space.wrap('y')) is None + assert not api.PyErr_Occurred() + + # getattr returns a bound method + assert not isinstance(api.PyObject_GetAttr(w_instance, space.wrap('f')), Function) + # _PyInstance_Lookup returns the raw descriptor + assert isinstance(api._PyInstance_Lookup(w_instance, space.wrap('f')), Function) From afa at codespeak.net Wed Apr 28 17:20:00 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 17:20:00 +0200 (CEST) Subject: [pypy-svn] r74182 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428152000.A59E8282B9D@codespeak.net> Author: afa Date: Wed Apr 28 17:19:59 2010 New Revision: 74182 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Log: PyList_AsTuple Modified: pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/listobject.py Wed Apr 28 17:19:59 2010 @@ -92,6 +92,12 @@ space.wrap("expected list object")) return PyList_GET_SIZE(space, ref) + at cpython_api([PyObject], PyObject) +def PyList_AsTuple(space, w_list): + """Return a new tuple object containing the contents of list; equivalent to + tuple(list).""" + return space.call_function(space.w_tuple, w_list) + @cpython_api([PyObject], rffi.INT_real, error=-1) def PyList_Sort(space, w_list): """Sort the items of list in place. Return 0 on success, -1 on Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 17:19:59 2010 @@ -2640,16 +2640,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyList_AsTuple(space, list): - """ - - - - Return a new tuple object containing the contents of list; equivalent to - tuple(list).""" - raise NotImplementedError - @cpython_api([Py_ssize_t], PyObject) def PyLong_FromSsize_t(space, v): """Return a new PyLongObject object from a C Py_ssize_t, or Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_listobject.py Wed Apr 28 17:19:59 2010 @@ -53,6 +53,11 @@ assert api.PyList_Reverse(l) == 0 assert space.eq_w(l, space.newlist([space.wrap(1), space.wrap(2), space.wrap(3)])) + def test_list_tuple(self, space, api): + w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)]) + w_t = api.PyList_AsTuple(w_l) + assert space.unwrap(w_t) == (3, 2, 1) + class AppTestListObject(AppTestCpythonExtensionBase): def test_listobject(self): import sys From afa at codespeak.net Wed Apr 28 17:21:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 17:21:23 +0200 (CEST) Subject: [pypy-svn] r74183 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428152123.87DF7282B9D@codespeak.net> Author: afa Date: Wed Apr 28 17:21:21 2010 New Revision: 74183 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: Already implemented Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 17:21:21 2010 @@ -1111,11 +1111,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], lltype.Void) -def PyDict_Clear(space, p): - """Empty an existing dictionary of all key-value pairs.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_Contains(space, p, key): """Determine if dictionary p contains key. If an item in p is matches From afa at codespeak.net Wed Apr 28 17:38:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 17:38:24 +0200 (CEST) Subject: [pypy-svn] r74184 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include test Message-ID: <20100428153824.B54A3282B9D@codespeak.net> Author: afa Date: Wed Apr 28 17:38:23 2010 New Revision: 74184 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/methodobject.h pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Log: Export fields of PyCFunction objects: ((PyCFunctionObject*)func)->m_ml->ml_name Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/methodobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/methodobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/methodobject.h Wed Apr 28 17:38:23 2010 @@ -21,6 +21,13 @@ }; typedef struct PyMethodDef PyMethodDef; +typedef struct +{ + PyObject_HEAD + PyMethodDef *m_ml; /* Description of the C function to call */ + PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */ +} PyCFunctionObject; + /* Flag passed to newmethodobject */ #define METH_OLDARGS 0x0000 #define METH_VARARGS 0x0001 @@ -42,7 +49,14 @@ #define METH_COEXIST 0x0040 - +/* Macros for direct access to these values. Type checks are *not* + done, so use with care. */ +#define PyCFunction_GET_FUNCTION(func) \ + (((PyCFunctionObject *)func) -> m_ml -> ml_meth) +#define PyCFunction_GET_SELF(func) \ + (((PyCFunctionObject *)func) -> m_self) +#define PyCFunction_GET_FLAGS(func) \ + (((PyCFunctionObject *)func) -> m_ml -> ml_flags) #ifdef __cplusplus } Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Wed Apr 28 17:38:23 2010 @@ -7,11 +7,12 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.function import BuiltinFunction, Method from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref +from pypy.module.cpyext.pyobject import (PyObject, from_ref, make_ref, make_typedescr, + Py_DecRef) from pypy.module.cpyext.api import ( generic_cpy_call, cpython_api, PyObject, cpython_struct, METH_KEYWORDS, METH_O, CONST_STRING, METH_CLASS, METH_STATIC, METH_COEXIST, METH_NOARGS, - METH_VARARGS, build_type_checkers) + METH_VARARGS, build_type_checkers, PyObjectFields, bootstrap_function) from pypy.module.cpyext.state import State from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.rlib.objectmodel import we_are_translated @@ -28,6 +29,37 @@ ('ml_doc', rffi.CCHARP), ]) +PyCFunctionObjectStruct = cpython_struct( + 'PyCFunctionObject', + PyObjectFields + ( + ('m_ml', lltype.Ptr(PyMethodDef)), + ('m_self', PyObject), + )) +PyCFunctionObject = lltype.Ptr(PyCFunctionObjectStruct) + + at bootstrap_function +def init_methodobject(space): + make_typedescr(W_PyCFunctionObject.typedef, + basestruct=PyCFunctionObject.TO, + attach=cfunction_attach, + dealloc=cfunction_dealloc) + +def cfunction_attach(space, py_obj, w_obj): + py_func = rffi.cast(PyCFunctionObject, py_obj) + assert isinstance(w_obj, W_PyCFunctionObject) + py_func.c_m_ml = w_obj.ml + py_func.c_m_self = make_ref(space, w_obj.w_self) + + at cpython_api([PyObject], lltype.Void, external=False) +def cfunction_dealloc(space, py_obj): + py_func = rffi.cast(PyCFunctionObject, py_obj) + Py_DecRef(space, py_func.c_m_self) + # standard dealloc + pto = py_obj.c_ob_type + obj_voidp = rffi.cast(rffi.VOIDP_real, py_obj) + generic_cpy_call(space, pto.c_tp_free, obj_voidp) + pto = rffi.cast(PyObject, pto) + Py_DecRef(space, pto) class W_PyCFunctionObject(Wrappable): def __init__(self, space, ml, w_self, doc=None): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_methodobject.py Wed Apr 28 17:38:23 2010 @@ -37,7 +37,8 @@ ('isCFunction', 'METH_O', ''' if(PyCFunction_Check(args)) { - Py_RETURN_TRUE; + PyCFunctionObject* func = (PyCFunctionObject*)args; + return PyString_FromString(func->m_ml->ml_name); } else { Py_RETURN_FALSE; @@ -57,4 +58,4 @@ assert mod.getarg_OLD() is None assert mod.getarg_OLD(1, 2) == (1, 2) - assert mod.isCFunction(mod.getarg_O) + assert mod.isCFunction(mod.getarg_O) == "getarg_O" From arigo at codespeak.net Wed Apr 28 18:33:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 18:33:31 +0200 (CEST) Subject: [pypy-svn] r74187 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp metainterp/test Message-ID: <20100428163331.E8FE9282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 18:33:30 2010 New Revision: 74187 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Enhance the tests to also run on top of pyjitpl. Start fixing pyjitpl. Three tests pass so far. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Wed Apr 28 18:33:30 2010 @@ -52,6 +52,7 @@ return jitcode def make_jitcodes(self, maingraph, verbose=False): + self.portal_graph = maingraph return self.transform_graph_to_jitcode(maingraph, verbose) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Wed Apr 28 18:33:30 2010 @@ -4,9 +4,11 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print +from pypy.tool.sourcetools import func_with_new_name from pypy.jit.metainterp import history, compile, resume -from pypy.jit.metainterp.history import Const, ConstInt, Box +from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstFloat +from pypy.jit.metainterp.history import Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import executor from pypy.jit.metainterp.logger import Logger @@ -17,6 +19,7 @@ from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.metainterp.compile import GiveUp +from pypy.jit.codewriter.assembler import JitCode # ____________________________________________________________ @@ -24,66 +27,11 @@ for arg in args: assert isinstance(arg, (Box, Const)) -class arguments(object): - def __init__(self, *argtypes): - self.argtypes = argtypes - - def __eq__(self, other): - if not isinstance(other, arguments): - return NotImplemented - return self.argtypes == other.argtypes - - def __ne__(self, other): - if not isinstance(other, arguments): - return NotImplemented - return self.argtypes != other.argtypes - - def __call__(self, func): - argtypes = unrolling_iterable(self.argtypes) - def wrapped(self, orgpc): - args = (self, ) - for argspec in argtypes: - if argspec == "box": - box = self.load_arg() - args += (box, ) - elif argspec == "constbox": - args += (self.load_const_arg(), ) - elif argspec == "int": - args += (self.load_int(), ) - elif argspec == "jumptarget": - args += (self.load_3byte(), ) - elif argspec == "jumptargets": - num = self.load_int() - args += ([self.load_3byte() for i in range(num)], ) - elif argspec == "varargs": - args += (self.load_varargs(), ) - elif argspec == "constargs": - args += (self.load_constargs(), ) - elif argspec == "descr": - descr = self.load_const_arg() - assert isinstance(descr, history.AbstractDescr) - args += (descr, ) - elif argspec == "bytecode": - bytecode = self.load_const_arg() - assert isinstance(bytecode, codewriter.JitCode) - args += (bytecode, ) - elif argspec == "orgpc": - args += (orgpc, ) - elif argspec == "methdescr": - methdescr = self.load_const_arg() - assert isinstance(methdescr, - history.AbstractMethDescr) - args += (methdescr, ) - else: - assert 0, "unknown argtype declaration: %r" % (argspec,) - val = func(*args) - if val is None: - val = False - return val - name = func.func_name - wrapped.func_name = "wrap_" + name - wrapped.argspec = self - return wrapped +def arguments(*args): + def decorate(func): + func.argtypes = args + return func + return decorate # ____________________________________________________________ @@ -95,16 +43,35 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - def __init__(self, metainterp, jitcode, greenkey=None): - assert isinstance(jitcode, codewriter.JitCode) + def __init__(self, metainterp): self.metainterp = metainterp + self.boxes_i = [None] * 256 + self.boxes_r = [None] * 256 + self.boxes_f = [None] * 256 + + def setup(self, jitcode, greenkey=None): + assert isinstance(jitcode, JitCode) self.jitcode = jitcode self.bytecode = jitcode.code - self.constants = jitcode.constants - self.exception_target = -1 self.name = jitcode.name # purely for having name attribute # this is not None for frames that are recursive portal calls self.greenkey = greenkey + # copy the constants in place + self.copy_constants(self.boxes_i, jitcode.constants_i, ConstInt) + self.copy_constants(self.boxes_r, jitcode.constants_r, ConstPtr) + self.copy_constants(self.boxes_f, jitcode.constants_f, ConstFloat) + + def copy_constants(self, boxes, constants, ConstClass): + """Copy jitcode.constants[0] to boxes[255], + jitcode.constants[1] to boxes[254], + jitcode.constants[2] to boxes[253], etc.""" + i = len(constants) - 1 + while i >= 0: + j = 255 - i + assert j >= 0 + boxes[j] = ConstClass(constants[i]) + i -= 1 + copy_constants._annspecialcase_ = 'specialize:arg(3)' # ------------------------------ # Decoding of the JitCode @@ -197,7 +164,7 @@ exec py.code.Source(''' @arguments("box", "box") def opimpl_%s(self, b1, b2): - self.execute(rop.%s, b1, b2) + return self.execute(rop.%s, b1, b2) ''' % (_opimpl, _opimpl.upper())).compile() for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: @@ -219,14 +186,16 @@ self.execute(rop.%s, b) ''' % (_opimpl, _opimpl.upper())).compile() - @arguments() - def opimpl_return(self): - assert len(self.env) == 1 - return self.metainterp.finishframe(self.env[0]) + @arguments("box") + def opimpl_any_return(self, box): + return self.metainterp.finishframe(box) + + opimpl_int_return = opimpl_any_return + opimpl_ref_return = opimpl_any_return + opimpl_float_return = opimpl_any_return @arguments() def opimpl_void_return(self): - assert len(self.env) == 0 return self.metainterp.finishframe(None) @arguments("jumptarget") @@ -962,16 +931,15 @@ # whenever the 'opcode_implementations' (which is one of the 'opimpl_' # methods) returns True. This is the case when the current frame # changes, due to a call or a return. - while True: - pc = self.pc - op = ord(self.bytecode[pc]) - #print self.metainterp.opcode_names[op] - self.pc = pc + 1 + try: staticdata = self.metainterp.staticdata - stop = staticdata.opcode_implementations[op](self, pc) - #self.metainterp.most_recent_mp = None - if stop: - break + while True: + pc = self.pc + op = ord(self.bytecode[pc]) + #print staticdata.opcode_names[op] + staticdata.opcode_implementations[op](self, pc) + except ChangeFrame: + pass def generate_guard(self, pc, opnum, box, extraargs=[]): if isinstance(box, Const): # no need for a guard @@ -1023,13 +991,11 @@ @specialize.arg(1) def execute(self, opnum, *argboxes): - self.execute_with_descr(opnum, None, *argboxes) + return self.metainterp.execute_and_record(opnum, None, *argboxes) @specialize.arg(1) def execute_with_descr(self, opnum, descr, *argboxes): - resbox = self.metainterp.execute_and_record(opnum, descr, *argboxes) - if resbox is not None: - self.make_result_box(resbox) + return self.metainterp.execute_and_record(opnum, descr, *argboxes) @specialize.arg(1) def execute_varargs(self, opnum, argboxes, descr, exc): @@ -1068,20 +1034,19 @@ logger_noopt = None logger_ops = None - def __init__(self, portal_graph, cpu, stats, options, + def __init__(self, codewriter, options, ProfilerClass=EmptyProfiler, warmrunnerdesc=None): - self.cpu = cpu - self.stats = stats + self.cpu = codewriter.cpu + self.stats = self.cpu.stats self.options = options self.logger_noopt = Logger(self) self.logger_ops = Logger(self, guard_number=True) - RESULT = portal_graph.getreturnvar().concretetype + RESULT = codewriter.portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) - self.opcode_implementations = [] - self.opcode_names = [] - self.opname_to_index = {} + self.setup_insns(codewriter.assembler.insns) + self.setup_descrs(codewriter.assembler.descrs) self.profiler = ProfilerClass() @@ -1089,16 +1054,15 @@ self.indirectcall_values = [] self.warmrunnerdesc = warmrunnerdesc - self._op_goto_if_not = self.find_opcode('goto_if_not') - self._op_ooisnull = self.find_opcode('ooisnull') - self._op_oononnull = self.find_opcode('oononnull') + #self._op_goto_if_not = self.find_opcode('goto_if_not') + #self._op_ooisnull = self.find_opcode('ooisnull') + #self._op_oononnull = self.find_opcode('oononnull') backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] self.jit_starting_line = 'JIT starting (%s)' % backendmodule self.portal_code = None - self._class_sizes = None self._addr2name_keys = [] self._addr2name_values = [] @@ -1110,13 +1074,18 @@ def _freeze_(self): return True - def info_from_codewriter(self, portal_code, class_sizes, - list_of_addr2name, portal_runner_ptr): - self.portal_code = portal_code - self._class_sizes = class_sizes - self._addr2name_keys = [key for key, value in list_of_addr2name] - self._addr2name_values = [value for key, value in list_of_addr2name] - self._portal_runner_ptr = portal_runner_ptr + def setup_insns(self, insns): + self.opcode_names = ['?'] * len(insns) + self.opcode_implementations = [None] * len(insns) + for key, value in insns.items(): + assert self.opcode_implementations[value] is None + self.opcode_names[value] = key + name, argcodes = key.split('/') + opimpl = _get_opimpl_method(name, argcodes) + self.opcode_implementations[value] = opimpl + + def setup_descrs(self, descrs): + self.opcode_descrs = descrs def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1134,20 +1103,12 @@ """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: debug_print(self.jit_starting_line) - self._setup_class_sizes() self.cpu.setup_once() if not self.profiler.initialized: self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - def _setup_class_sizes(self): - class_sizes = {} - for vtable, sizedescr in self._class_sizes: - vtable = self.cpu.ts.cast_vtable_to_hashable(self.cpu, vtable) - class_sizes[vtable] = sizedescr - self.cpu.set_class_sizes(class_sizes) - def get_name_from_address(self, addr): # for debugging only if we_are_translated(): @@ -1196,21 +1157,6 @@ self.indirectcall_keys.append(fnaddress) self.indirectcall_values.append(jitcode) - def find_opcode(self, name): - try: - return self.opname_to_index[name] - except KeyError: - self._register_opcode(name) - return self.opname_to_index[name] - - def _register_opcode(self, opname): - assert len(self.opcode_implementations) < 256, \ - "too many implementations of opcodes!" - name = "opimpl_" + opname - self.opname_to_index[opname] = len(self.opcode_implementations) - self.opcode_names.append(opname) - self.opcode_implementations.append(getattr(MIFrame, name).im_func) - # ---------------- logging ------------------------ def log(self, msg): @@ -1268,6 +1214,7 @@ self.cpu = staticdata.cpu self.portal_trace_positions = [] self.greenkey_of_huge_function = None + self.free_frames_list = [] def is_blackholing(self): return self.history is None @@ -1278,7 +1225,11 @@ if greenkey is not None and not self.is_blackholing(): self.portal_trace_positions.append( (greenkey, len(self.history.operations))) - f = MIFrame(self, jitcode, greenkey) + if len(self.free_frames_list) > 0: + f = self.free_frames_list.pop() + else: + f = MIFrame(self) + f.setup(jitcode, greenkey) self.framestack.append(f) return f @@ -1289,10 +1240,13 @@ if frame.greenkey is not None and not self.is_blackholing(): self.portal_trace_positions.append( (None, len(self.history.operations))) - return frame + # we save the freed MIFrames to avoid needing to re-create new + # MIFrame objects all the time; they are a bit big, with their + # 3*256 register entries. + self.free_frames_list.append(frame) def finishframe(self, resultbox): - frame = self.popframe() + self.popframe() if self.framestack: if resultbox is not None: self.framestack[-1].make_result_box(resultbox) @@ -1501,7 +1455,7 @@ except: import sys if sys.exc_info()[0] is not None: - codewriter.log.info(sys.exc_info()[0].__name__) + self.staticdata.log(sys.exc_info()[0].__name__) raise def compile_and_run_once(self, *args): @@ -1739,7 +1693,19 @@ self.framestack = [] f = self.newframe(self.staticdata.portal_code) f.pc = 0 - f.env = original_boxes[:] + count_i = count_r = count_f = 0 + for box in original_boxes: + if box.type == history.INT: + f.boxes_i[count_i] = box + count_i += 1 + elif box.type == history.REF: + f.boxes_r[count_r] = box + count_r += 1 + elif box.type == history.FLOAT: + f.boxes_f[count_f] = box + count_f += 1 + else: + raise AssertionError(box.type) self.virtualref_boxes = [] self.initialize_virtualizable(original_boxes) return original_boxes @@ -2045,3 +2011,64 @@ assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token + +# ____________________________________________________________ + +class ChangeFrame(Exception): + pass + +def _get_opimpl_method(name, argcodes): + from pypy.jit.metainterp.blackhole import signedord + # + def handler(self, position): + args = () + next_argcode = 0 + code = self.bytecode + position += 1 + for argtype in argtypes: + if argtype == "box": + argcode = argcodes[next_argcode] + next_argcode = next_argcode + 1 + if argcode == 'i': + value = self.boxes_i[ord(code[position])] + elif argcode == 'c': + value = ConstInt(signedord(code[position])) + elif argcode == 'r': + value = self.boxes_r[ord(code[position])] + elif argcode == 'f': + value = self.boxes_f[ord(code[position])] + else: + raise AssertionError("bad argcode") + position += 1 + else: + raise AssertionError("bad argtype: %r" % (argtype,)) + args += (value,) + # + num_return_args = len(argcodes) - next_argcode + assert num_return_args == 0 or num_return_args == 1 + self.pc = position + num_return_args + # + resultbox = unboundmethod(self, *args) + # + if num_return_args == 0: + assert resultbox is None + else: + assert resultbox is not None + result_argcode = argcodes[next_argcode] + target_index = ord(code[position]) + if result_argcode == 'i': + assert resultbox.type == history.INT + self.boxes_i[target_index] = resultbox + elif result_argcode == 'r': + assert resultbox.type == history.REF + self.boxes_r[target_index] = resultbox + elif result_argcode == 'f': + assert resultbox.type == history.FLOAT + self.boxes_f[target_index] = resultbox + else: + raise AssertionError("bad result argcode") + # + unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func + argtypes = unrolling_iterable(unboundmethod.argtypes) + handler = func_with_new_name(handler, 'handler_' + name) + return handler Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Wed Apr 28 18:33:30 2010 @@ -15,6 +15,7 @@ def _get_jitcodes(CPUClass, func, values, type_system): from pypy.jit.codewriter import support, codewriter + func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) graphs = rtyper.annotator.translator.graphs stats = history.Stats() @@ -44,21 +45,51 @@ blackholeinterp.run(mainjitcode, 0) return blackholeinterp.result_i -def _get_bare_metainterp(func, values, CPUClass, type_system): - from pypy.annotation.policy import AnnotatorPolicy - from pypy.annotation.model import lltype_to_annotation - from pypy.rpython.test.test_llinterp import gengraph +def _run_with_pyjitpl(cw, mainjitcode, args, testself): + from pypy.jit.metainterp import simple_optimize - rtyper = support.annotate(func, values, type_system=type_system) + class DoneWithThisFrame(Exception): + pass - stats = history.Stats() - cpu = CPUClass(rtyper, stats, None, False) - graphs = rtyper.annotator.translator.graphs - opt = history.Options(listops=listops) - metainterp_sd = pyjitpl.MetaInterpStaticData(graphs[0], cpu, stats, opt) + class DoneWithThisFrameRef(DoneWithThisFrame): + def __init__(self, cpu, *args): + DoneWithThisFrame.__init__(self, *args) + + class FakeWarmRunnerState: + def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): + pass + + # pick the optimizer this way + optimize_loop = staticmethod(simple_optimize.optimize_loop) + optimize_bridge = staticmethod(simple_optimize.optimize_bridge) + + trace_limit = sys.maxint + debug_level = 2 + + opt = history.Options(listops=True) + cpu = cw.cpu + metainterp_sd = pyjitpl.MetaInterpStaticData(cw, opt) metainterp_sd.finish_setup(optimizer="bogus") + metainterp_sd.state = FakeWarmRunnerState() + metainterp_sd.state.cpu = metainterp_sd.cpu metainterp = pyjitpl.MetaInterp(metainterp_sd) - return metainterp, rtyper + if hasattr(testself, 'finish_metainterp_for_interp_operations'): + testself.finish_metainterp_for_interp_operations(metainterp) + + metainterp_sd.portal_code = mainjitcode + metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrame + metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef + metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrame + testself.metainterp = metainterp + try: + metainterp.compile_and_run_once(*args) + except DoneWithThisFrame, e: + #if conftest.option.view: + # metainterp.stats.view() + return e.args[0] + else: + raise Exception("FAILED") + class JitMixin: basic = True @@ -99,61 +130,11 @@ cw, mainjitcode = _get_jitcodes(self.CPUClass, f, args, self.type_system) # try to run it with blackhole.py - result = _run_with_blackhole(cw, mainjitcode, args) + result1 = _run_with_blackhole(cw, mainjitcode, args) # try to run it with pyjitpl.py - # -- XXX --- missing - return result - - from pypy.jit.metainterp import simple_optimize - - class DoneWithThisFrame(Exception): - pass - - class DoneWithThisFrameRef(DoneWithThisFrame): - def __init__(self, cpu, *args): - DoneWithThisFrame.__init__(self, *args) - - class FakeWarmRunnerState: - def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): - pass - - # pick the optimizer this way - optimize_loop = staticmethod(simple_optimize.optimize_loop) - optimize_bridge = staticmethod(simple_optimize.optimize_bridge) - - trace_limit = sys.maxint - debug_level = 2 - - metainterp, rtyper = _get_bare_metainterp(f, args, self.CPUClass, - self.type_system, - **kwds) - metainterp.staticdata.state = FakeWarmRunnerState() - metainterp.staticdata.state.cpu = metainterp.staticdata.cpu - if hasattr(self, 'finish_metainterp_for_interp_operations'): - self.finish_metainterp_for_interp_operations(metainterp) - portal_graph = rtyper.annotator.translator.graphs[0] - cw = codewriter.CodeWriter(rtyper) - - graphs = cw.find_all_graphs(portal_graph, JitPolicy(), - self.CPUClass.supports_floats) - cw._start(metainterp.staticdata, None) - portal_graph.func._jit_unroll_safe_ = True - maingraph = cw.make_one_bytecode((portal_graph, None), False) - cw.finish_making_bytecodes() - metainterp.staticdata.portal_code = maingraph - metainterp.staticdata._class_sizes = cw.class_sizes - metainterp.staticdata.DoneWithThisFrameInt = DoneWithThisFrame - metainterp.staticdata.DoneWithThisFrameRef = DoneWithThisFrameRef - metainterp.staticdata.DoneWithThisFrameFloat = DoneWithThisFrame - self.metainterp = metainterp - try: - metainterp.compile_and_run_once(*args) - except DoneWithThisFrame, e: - #if conftest.option.view: - # metainterp.stats.view() - return e.args[0] - else: - raise Exception("FAILED") + result2 = _run_with_pyjitpl(cw, mainjitcode, args, self) + assert result1 == result2 + return result1 def check_history(self, expected=None, **isns): # this can be used after calling meta_interp From arigo at codespeak.net Wed Apr 28 19:01:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Apr 2010 19:01:27 +0200 (CEST) Subject: [pypy-svn] r74188 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100428170127.6500C282B9D@codespeak.net> Author: arigo Date: Wed Apr 28 19:01:25 2010 New Revision: 74188 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Log: Add an extra byte at the end of the bytecode to mean "the highest register number used, among registers of type 'r'". Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Wed Apr 28 19:01:25 2010 @@ -18,13 +18,25 @@ self.called_from = called_from self.graph = graph - def setup(self, code, constants_i=[], constants_r=[], constants_f=[]): - self.code = code + def setup(self, code, constants_i=[], constants_r=[], constants_f=[], + num_regs_r=256): + # stick an extra char at the end of self.code, which is the + # highest 'r' register used in this code. It default to 255, + # which is always correct. Also, even if no 'r' register is + # used it must be set to 0, which means that register %r0 is + # always marked as used. + self.code = code + chr((num_regs_r or 1)-1) # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i self.constants_r = constants_r or self._empty_r self.constants_f = constants_f or self._empty_f + def _code(self): + return self.code[:-1] # for testing, without the extra char + + def highest_r_reg(self): + return ord(self.code[-1]) + class Assembler(object): @@ -50,11 +62,11 @@ self.label_positions = {} self.tlabel_positions = [] self.switchdictdescrs = [] - self.highest_regs = dict.fromkeys(KINDS, 0) + self.count_regs = dict.fromkeys(KINDS, 0) def emit_reg(self, reg): - if reg.index > self.highest_regs[reg.kind]: - self.highest_regs[reg.kind] = reg.index + if reg.index >= self.count_regs[reg.kind]: + self.count_regs[reg.kind] = reg.index + 1 self.code.append(chr(reg.index)) def emit_const(self, const, kind, allow_short=False): @@ -164,14 +176,15 @@ def check_result(self): # Limitation of the number of registers, from the single-byte encoding - assert self.highest_regs['int'] + len(self.constants_i) <= 256 - assert self.highest_regs['ref'] + len(self.constants_r) <= 256 - assert self.highest_regs['float'] + len(self.constants_f) <= 256 + assert self.count_regs['int'] + len(self.constants_i) <= 256 + assert self.count_regs['ref'] + len(self.constants_r) <= 256 + assert self.count_regs['float'] + len(self.constants_f) <= 256 def make_jitcode(self, name): jitcode = JitCode(name) jitcode.setup(''.join(self.code), self.constants_i, self.constants_r, - self.constants_f) + self.constants_f, + self.count_regs['ref']) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Wed Apr 28 19:01:25 2010 @@ -17,7 +17,8 @@ assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x00\x01\x02" - "\x01\x02") + "\x01\x02" + "\x00") assert assembler.insns == {'int_add/iii': 0, 'int_return/i': 1} @@ -36,7 +37,8 @@ "\x01\x12" # use int_return/c for one-byte consts "\x01\xFC" "\x00\xFF" # use int_return/i for larger consts - "\x00\xFE") + "\x00\xFE" + "\x00") # highest_r_reg assert assembler.insns == {'int_return/i': 0, 'int_return/c': 1} assert jitcode.constants_i == [128, -129] @@ -54,7 +56,8 @@ assert jitcode.code == ("\x00\x0D" "\x00\xFF" "\x00\xFE" - "\x00\xFD") + "\x00\xFD" + "\x00") assert assembler.insns == {'float_return/f': 0} assert jitcode.constants_f == [18.0, -4.0, 128.1] @@ -75,7 +78,8 @@ assert jitcode.code == ("\x00\x58" "\x01\xFF" "\x01\xFE" - "\x02\xFF") + "\x02\xFF" + "\x00") assert assembler.insns == {'int_return/c': 0, 'int_return/i': 1, 'ref_return/r': 2} @@ -102,7 +106,8 @@ "\x01\x17\x16\x17" "\x02\x16\x01\x16" "\x03\x00\x00" - "\x04\x17") + "\x04\x17" + "\x00") assert assembler.insns == {'goto_if_not_int_gt/Lic': 0, 'int_add/iii': 1, 'int_sub/ici': 2, @@ -118,7 +123,7 @@ ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode.code == "\x00\x03\x16\x17\xFF\x00" + assert jitcode._code() == "\x00\x03\x16\x17\xFF\x00" assert assembler.insns == {'foobar/IR': 0} assert jitcode.constants_i == [42] @@ -130,7 +135,18 @@ ssarepr.insns = [('foobar', d) for d in descrs[::-1]] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode.code == ''.join(["\x00" + struct.pack(" Author: afa Date: Wed Apr 28 19:03:57 2010 New Revision: 74189 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: Most types now call PyObject_dealloc() instead of copying the same code again and again For some reason this did not work with type_dealloc, why? Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Wed Apr 28 19:03:57 2010 @@ -30,12 +30,8 @@ def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) - # standard dealloc - pto = py_obj.c_ob_type - obj_voidp = rffi.cast(rffi.VOIDP_real, py_obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) @cpython_api([PyObject], PyObject, borrowed=True) def PyMethod_Function(space, w_method): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/methodobject.py Wed Apr 28 19:03:57 2010 @@ -54,12 +54,8 @@ def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) - # standard dealloc - pto = py_obj.c_ob_type - obj_voidp = rffi.cast(rffi.VOIDP_real, py_obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) class W_PyCFunctionObject(Wrappable): def __init__(self, space, ml, w_self, doc=None): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 19:03:57 2010 @@ -31,11 +31,11 @@ @cpython_api([PyObject], lltype.Void) def PyObject_dealloc(space, obj): - pto = rffi.cast(PyTypeObjectPtr, obj.c_ob_type) + pto = obj.c_ob_type obj_voidp = rffi.cast(rffi.VOIDP_real, obj) generic_cpy_call(space, pto.c_tp_free, obj_voidp) if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, rffi.cast(PyObject, obj.c_ob_type)) + Py_DecRef(space, rffi.cast(PyObject, pto)) @cpython_api([PyTypeObjectPtr], PyObject) def _PyObject_GC_New(space, type): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stringobject.py Wed Apr 28 19:03:57 2010 @@ -51,15 +51,12 @@ return w_str @cpython_api([PyObject], lltype.Void, external=False) -def string_dealloc(space, obj): - obj = rffi.cast(PyStringObject, obj) - pto = obj.c_ob_type - if obj.c_buffer: - lltype.free(obj.c_buffer, flavor="raw") - obj_voidp = rffi.cast(rffi.VOIDP_real, obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) +def string_dealloc(space, py_obj): + py_str = rffi.cast(PyStringObject, py_obj) + if py_str.c_buffer: + lltype.free(py_str.c_buffer, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) @cpython_api([CONST_STRING, Py_ssize_t], PyObject) def PyString_FromStringAndSize(space, char_p, length): Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 28 19:03:57 2010 @@ -40,15 +40,12 @@ py_unicode.c_buffer = lltype.nullptr(rffi.CWCHARP.TO) @cpython_api([PyObject], lltype.Void, external=False) -def unicode_dealloc(space, obj): - obj = rffi.cast(PyUnicodeObject, obj) - pto = obj.c_ob_type - if obj.c_buffer: - lltype.free(obj.c_buffer, flavor="raw") - obj_voidp = rffi.cast(rffi.VOIDP_real, obj) - generic_cpy_call(space, pto.c_tp_free, obj_voidp) - pto = rffi.cast(PyObject, pto) - Py_DecRef(space, pto) +def unicode_dealloc(space, py_obj): + py_unicode = rffi.cast(PyUnicodeObject, py_obj) + if py_unicode.c_buffer: + lltype.free(py_unicode.c_buffer, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) @cpython_api([Py_UNICODE], rffi.INT_real, error=CANNOT_FAIL) def Py_UNICODE_ISSPACE(space, ch): From afa at codespeak.net Wed Apr 28 19:28:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:28:46 +0200 (CEST) Subject: [pypy-svn] r74190 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428172846.392AB282B9E@codespeak.net> Author: afa Date: Wed Apr 28 19:28:44 2010 New Revision: 74190 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Log: PyUnicode_FromEncodedObject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 19:28:44 2010 @@ -3904,23 +3904,6 @@ possible. This macro does not raise exceptions.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject) -def PyUnicode_FromEncodedObject(space, obj, encoding, errors): - """Coerce an encoded object obj to an Unicode object and return a reference with - incremented refcount. - - String and other char buffer compatible objects are decoded according to the - given encoding and using the error handling defined by errors. Both can be - NULL to have the interface use the default values (see the next section for - details). - - All other objects, including Unicode objects, cause a TypeError to be - set. - - The API returns NULL if there was an error. The caller is responsible for - decref'ing the returned objects.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyUnicode_FromObject(space, obj): """Shortcut for PyUnicode_FromEncodedObject(obj, NULL, "strict") which is used Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_unicodeobject.py Wed Apr 28 19:28:44 2010 @@ -94,6 +94,17 @@ b_encoding = rffi.str2charp('cp437') assert space.unwrap( api.PyUnicode_Decode(b_text, 4, b_encoding, None)) == u'caf\xe9' + + w_text = api.PyUnicode_FromEncodedObject(space.wrap("test"), b_encoding, None) + assert space.is_true(space.isinstance(w_text, space.w_unicode)) + assert space.unwrap(w_text) == "test" + + assert api.PyUnicode_FromEncodedObject(space.wrap(u"test"), b_encoding, None) is None + assert api.PyErr_Occurred() is space.w_TypeError + assert api.PyUnicode_FromEncodedObject(space.wrap(1), b_encoding, None) is None + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + rffi.free_charp(b_text) rffi.free_charp(b_encoding) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/unicodeobject.py Wed Apr 28 19:28:44 2010 @@ -243,6 +243,39 @@ w_errors = space.w_None return space.call_method(w_str, 'decode', w_encoding, w_errors) + at cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) +def PyUnicode_FromEncodedObject(space, w_obj, encoding, errors): + """Coerce an encoded object obj to an Unicode object and return a reference with + incremented refcount. + + String and other char buffer compatible objects are decoded according to the + given encoding and using the error handling defined by errors. Both can be + NULL to have the interface use the default values (see the next section for + details). + + All other objects, including Unicode objects, cause a TypeError to be + set.""" + w_encoding = space.wrap(rffi.charp2str(encoding)) + if errors: + w_errors = space.wrap(rffi.charp2str(errors)) + else: + w_errors = space.w_None + + # - unicode is disallowed + # - raise TypeError for non-string types + if space.is_true(space.isinstance(w_obj, space.w_unicode)): + w_meth = None + else: + try: + w_meth = space.getattr(w_obj, space.wrap('decode')) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + w_meth = None + if w_meth is None: + raise OperationError(space.w_TypeError, + space.wrap("decoding Unicode is not supported")) + return space.call_function(w_meth, w_encoding, w_errors) if sys.platform == 'win32': @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject) From afa at codespeak.net Wed Apr 28 19:33:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:33:05 +0200 (CEST) Subject: [pypy-svn] r74191 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428173305.80853282B9E@codespeak.net> Author: afa Date: Wed Apr 28 19:33:03 2010 New Revision: 74191 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_Compare Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 19:33:03 2010 @@ -188,6 +188,16 @@ by reverse quotes.""" return space.repr(w_obj) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyObject_Compare(space, w_o1, w_o2): + """ + Compare the values of o1 and o2 using a routine provided by o1, if one + exists, otherwise with a routine provided by o2. Returns the result of the + comparison on success. On error, the value returned is undefined; use + PyErr_Occurred() to detect an error. This is equivalent to the Python + expression cmp(o1, o2).""" + return space.int_w(space.cmp(w_o1, w_o2)) + @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject) def PyObject_RichCompare(space, w_o1, w_o2, opid_int): """Compare the values of o1 and o2 using the operation specified by opid, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 19:33:03 2010 @@ -2968,16 +2968,6 @@ the Python statement result = cmp(o1, o2).""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=None) -def PyObject_Compare(space, o1, o2): - """ - Compare the values of o1 and o2 using a routine provided by o1, if one - exists, otherwise with a routine provided by o2. Returns the result of the - comparison on success. On error, the value returned is undefined; use - PyErr_Occurred() to detect an error. This is equivalent to the Python - expression cmp(o1, o2).""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyObject_Bytes(space, o): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 28 19:33:03 2010 @@ -167,3 +167,8 @@ def test_type(self, space, api): assert api.PyObject_Type(space.wrap(72)) is space.w_int + + def test_compare(self, space, api): + assert api.PyObject_Compare(space.wrap(42), space.wrap(72)) == -1 + assert api.PyObject_Compare(space.wrap(72), space.wrap(42)) == 1 + assert api.PyObject_Compare(space.wrap("a"), space.wrap("a")) == 0 From afa at codespeak.net Wed Apr 28 19:36:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:36:18 +0200 (CEST) Subject: [pypy-svn] r74192 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428173618.2ADC2282B9E@codespeak.net> Author: afa Date: Wed Apr 28 19:36:16 2010 New Revision: 74192 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Log: PyObject_Unicode Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 19:36:16 2010 @@ -188,6 +188,14 @@ by reverse quotes.""" return space.repr(w_obj) + at cpython_api([PyObject], PyObject) +def PyObject_Unicode(space, w_obj): + """Compute a Unicode string representation of object o. Returns the Unicode + string representation on success, NULL on failure. This is the equivalent of + the Python expression unicode(o). Called by the unicode() built-in + function.""" + return space.call_function(space.w_unicode, w_obj) + @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_Compare(space, w_o1, w_o2): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 19:36:16 2010 @@ -2978,18 +2978,6 @@ for PyObject_Str().""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_Unicode(space, o): - """ - - - - Compute a Unicode string representation of object o. Returns the Unicode - string representation on success, NULL on failure. This is the equivalent of - the Python expression unicode(o). Called by the unicode() built-in - function.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, ], PyObject) def PyObject_CallFunction(space, callable, format, ): """ Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_object.py Wed Apr 28 19:36:16 2010 @@ -172,3 +172,9 @@ assert api.PyObject_Compare(space.wrap(42), space.wrap(72)) == -1 assert api.PyObject_Compare(space.wrap(72), space.wrap(42)) == 1 assert api.PyObject_Compare(space.wrap("a"), space.wrap("a")) == 0 + + def test_unicode(self, space, api): + assert space.unwrap(api.PyObject_Unicode(space.wrap([]))) == u"[]" + assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" + assert api.PyObject_Unicode(space.wrap("\xe9")) is None + api.PyErr_Clear() From afa at codespeak.net Wed Apr 28 19:43:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:43:06 +0200 (CEST) Subject: [pypy-svn] r74193 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428174306.7A57C282B9E@codespeak.net> Author: afa Date: Wed Apr 28 19:43:04 2010 New Revision: 74193 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: expose more functions already defined in bufferobject.c Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 28 19:43:04 2010 @@ -250,13 +250,17 @@ SYMBOLS_C = [ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', + '_PyArg_NoKeywords', 'PyString_FromFormat', 'PyString_FromFormatV', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', 'Py_BuildValue', 'PyTuple_Pack', 'PyErr_Format', 'PyErr_NewException', + 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', - 'PyBuffer_FromMemory', 'PyBuffer_New', 'PyBuffer_Type', 'init_bufferobject', - '_PyArg_NoKeywords', + + 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', + 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', 'init_bufferobject', + 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', ] TYPES = {} From afa at codespeak.net Wed Apr 28 19:47:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:47:24 +0200 (CEST) Subject: [pypy-svn] r74194 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428174724.F09C4282B9D@codespeak.net> Author: afa Date: Wed Apr 28 19:47:23 2010 New Revision: 74194 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Log: remove from stubs already implemented functions Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 19:47:23 2010 @@ -51,260 +51,18 @@ methods argument.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ], rffi.INT_real, error=0) -def PyArg_ParseTuple(space, args, format, ): - """Parse the parameters of a function that takes only positional parameters - into local variables. Returns true on success; on failure, it returns - false and raises the appropriate exception.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP, va_list], rffi.INT_real, error=0) def PyArg_VaParse(space, args, format, vargs): """Identical to PyArg_ParseTuple(), except that it accepts a va_list rather than a variable number of arguments.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, ], rffi.INT_real, error=0) -def PyArg_ParseTupleAndKeywords(space, args, kw, format, keywords, ): - """Parse the parameters of a function that takes both positional and keyword - parameters into local variables. Returns true on success; on failure, it - returns false and raises the appropriate exception.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, va_list], rffi.INT_real, error=0) def PyArg_VaParseTupleAndKeywords(space, args, kw, format, keywords, vargs): """Identical to PyArg_ParseTupleAndKeywords(), except that it accepts a va_list rather than a variable number of arguments.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ], rffi.INT_real, error=0) -def PyArg_Parse(space, args, format, ): - """Function used to deconstruct the argument lists of "old-style" functions - --- these are functions which use the METH_OLDARGS parameter - parsing method. This is not recommended for use in parameter parsing in - new code, and most code in the standard interpreter has been modified to no - longer use this for that purpose. It does remain a convenient way to - decompose other tuples, however, and may continue to be used for that - purpose.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, Py_ssize_t, Py_ssize_t, ], rffi.INT_real, error=0) -def PyArg_UnpackTuple(space, args, name, min, max, ): - """A simpler form of parameter retrieval which does not use a format string to - specify the types of the arguments. Functions which use this method to - retrieve their parameters should be declared as METH_VARARGS in - function or method tables. The tuple containing the actual parameters - should be passed as args; it must actually be a tuple. The length of the - tuple must be at least min and no more than max; min and max may be - equal. Additional arguments must be passed to the function, each of which - should be a pointer to a PyObject* variable; these will be filled - in with the values from args; they will contain borrowed references. The - variables which correspond to optional parameters not given by args will - not be filled in; these should be initialized by the caller. This function - returns true on success and false if args is not a tuple or contains the - wrong number of elements; an exception will be set if there was a failure. - - This is an example of the use of this function, taken from the sources for - the _weakref helper module for weak references: - - static PyObject * - weakref_ref(PyObject *self, PyObject *args) - { - PyObject *object; - PyObject *callback = NULL; - PyObject *result = NULL; - - if (PyArg_UnpackTuple(args, "ref", 1, 2, &object, &callback)) { - result = PyWeakref_NewRef(object, callback); - } - return result; - } - - The call to PyArg_UnpackTuple() in this example is entirely - equivalent to this call to PyArg_ParseTuple(): - - PyArg_ParseTuple(args, "O|O:ref", &object, &callback) - - - - This function used an int type for min and max. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, ], PyObject) -def Py_BuildValue(space, format, ): - """Create a new value based on a format string similar to those accepted by - the PyArg_Parse*() family of functions and a sequence of values. - Returns the value or NULL in the case of an error; an exception will be - raised if NULL is returned. - - Py_BuildValue() does not always build a tuple. It builds a tuple - only if its format string contains two or more format units. If the format - string is empty, it returns None; if it contains exactly one format - unit, it returns whatever object is described by that format unit. To - force it to return a tuple of size 0 or one, parenthesize the format - string. - - When memory buffers are passed as parameters to supply data to build - objects, as for the s and s# formats, the required data is copied. - Buffers provided by the caller are never referenced by the objects created - by Py_BuildValue(). In other words, if your code invokes - malloc() and passes the allocated memory to Py_BuildValue(), - your code is responsible for calling free() for that memory once - Py_BuildValue() returns. - - In the following description, the quoted form is the format unit; the entry - in (round) parentheses is the Python object type that the format unit will - return; and the entry in [square] brackets is the type of the C value(s) to - be passed. - - The characters space, tab, colon and comma are ignored in format strings - (but not within format units such as s#). This can be used to make - long format strings a tad more readable. - - s (string) [char *] - - Convert a null-terminated C string to a Python object. If the C string - pointer is NULL, None is used. - - s# (string) [char *, int] - - Convert a C string and its length to a Python object. If the C string - pointer is NULL, the length is ignored and None is returned. - - z (string or None) [char *] - - Same as s. - - z# (string or None) [char *, int] - - Same as s#. - - u (Unicode string) [Py_UNICODE *] - - Convert a null-terminated buffer of Unicode (UCS-2 or UCS-4) data to a - Python Unicode object. If the Unicode buffer pointer is NULL, - None is returned. - - u# (Unicode string) [Py_UNICODE *, int] - - Convert a Unicode (UCS-2 or UCS-4) data buffer and its length to a - Python Unicode object. If the Unicode buffer pointer is NULL, the - length is ignored and None is returned. - - i (integer) [int] - - Convert a plain C int to a Python integer object. - - b (integer) [char] - - Convert a plain C char to a Python integer object. - - h (integer) [short int] - - Convert a plain C short int to a Python integer object. - - l (integer) [long int] - - Convert a C long int to a Python integer object. - - B (integer) [unsigned char] - - Convert a C unsigned char to a Python integer object. - - H (integer) [unsigned short int] - - Convert a C unsigned short int to a Python integer object. - - I (integer/long) [unsigned int] - - Convert a C unsigned int to a Python integer object or a Python - long integer object, if it is larger than sys.maxint. - - k (integer/long) [unsigned long] - - Convert a C unsigned long to a Python integer object or a - Python long integer object, if it is larger than sys.maxint. - - L (long) [PY_LONG_LONG] - - Convert a C long long to a Python long integer object. Only - available on platforms that support long long. - - K (long) [unsigned PY_LONG_LONG] - - Convert a C unsigned long long to a Python long integer object. - Only available on platforms that support unsigned long long. - - n (int) [Py_ssize_t] - - Convert a C Py_ssize_t to a Python integer or long integer. - - - - c (string of length 1) [char] - - Convert a C int representing a character to a Python string of - length 1. - - d (float) [double] - - Convert a C double to a Python floating point number. - - f (float) [float] - - Same as d. - - D (complex) [Py_complex *] - - Convert a C Py_complex structure to a Python complex number. - - O (object) [PyObject *] - - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a NULL pointer, it is - assumed that this was caused because the call producing the argument - found an error and set an exception. Therefore, Py_BuildValue() - will return NULL but won't raise an exception. If no exception has - been raised yet, SystemError is set. - - S (object) [PyObject *] - - Same as O. - - N (object) [PyObject *] - - Same as O, except it doesn't increment the reference count on the - object. Useful when the object is created by a call to an object - constructor in the argument list. - - O& (object) [converter, anything] - - Convert anything to a Python object through a converter function. - The function is called with anything (which should be compatible with - void *) as its argument and should return a "new" Python - object, or NULL if an error occurred. - - (items) (tuple) [matching-items] - - Convert a sequence of C values to a Python tuple with the same number of - items. - - [items] (list) [matching-items] - - Convert a sequence of C values to a Python list with the same number of - items. - - {items} (dictionary) [matching-items] - - Convert a sequence of C values to a Python dictionary. Each pair of - consecutive C values adds one item to the dictionary, serving as key and - value, respectively. - - If there is an error in the format string, the SystemError exception - is set and NULL returned.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, va_list], PyObject) def Py_VaBuildValue(space, format, vargs): """Identical to Py_BuildValue(), except that it accepts a va_list @@ -507,71 +265,6 @@ raise NotImplementedError @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyBuffer_Check(space, p): - """Return true if the argument has type PyBuffer_Type.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) -def PyBuffer_FromObject(space, base, offset, size): - """Return a new read-only buffer object. This raises TypeError if - base doesn't support the read-only buffer protocol or doesn't provide - exactly one buffer segment, or it raises ValueError if offset is - less than zero. The buffer will hold a reference to the base object, and - the buffer's contents will refer to the base object's buffer interface, - starting as position offset and extending for size bytes. If size is - Py_END_OF_BUFFER, then the new buffer's contents extend to the - length of the base object's exported buffer data. - - This function used an int type for offset and size. This - might require changes in your code for properly supporting 64-bit - systems.""" - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) -def PyBuffer_FromReadWriteObject(space, base, offset, size): - """Return a new writable buffer object. Parameters and exceptions are similar - to those for PyBuffer_FromObject(). If the base object does not - export the writeable buffer protocol, then TypeError is raised. - - This function used an int type for offset and size. This - might require changes in your code for properly supporting 64-bit - systems.""" - raise NotImplementedError - - at cpython_api([rffi.VOIDP_real, Py_ssize_t], PyObject) -def PyBuffer_FromMemory(space, ptr, size): - """Return a new read-only buffer object that reads from a specified location - in memory, with a specified size. The caller is responsible for ensuring - that the memory buffer, passed in as ptr, is not deallocated while the - returned buffer object exists. Raises ValueError if size is less - than zero. Note that Py_END_OF_BUFFER may not be passed for the - size parameter; ValueError will be raised in that case. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([rffi.VOIDP_real, Py_ssize_t], PyObject) -def PyBuffer_FromReadWriteMemory(space, ptr, size): - """Similar to PyBuffer_FromMemory(), but the returned buffer is - writable. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([Py_ssize_t], PyObject) -def PyBuffer_New(space, size): - """Return a new writable buffer object that maintains its own memory buffer of - size bytes. ValueError is returned if size is not zero or - positive. Note that the memory buffer (as returned by - PyObject_AsWriteBuffer()) is not specifically aligned. - - This function used an int type for size. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyByteArray_Check(space, o): """Return true if the object o is a bytearray object or an instance of a subtype of the bytearray type.""" @@ -763,19 +456,6 @@ number object.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, ], rffi.INT_real, error=CANNOT_FAIL) -def PyOS_snprintf(space, str, size, format, ): - """Output not more than size bytes to str according to the format string - format and the extra arguments. See the Unix man page snprintf(2).""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, va_list], rffi.INT_real, error=CANNOT_FAIL) -def PyOS_vsnprintf(space, str, size, format, va): - """Output not more than size bytes to str according to the format string - format and the variable argument list va. Unix man page - vsnprintf(2).""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) def PyOS_string_to_double(space, s, endptr, overflow_exception): """Convert a string s to a double, raising a Python @@ -1203,154 +883,6 @@ The delayed normalization is implemented to improve performance.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ], PyObject) -def PyErr_Format(space, exception, format, ): - """This function sets the error indicator and returns NULL. exception should be - a Python exception (class, not an instance). format should be a string, - containing format codes, similar to printf(). The width.precision - before a format code is parsed, but the width part is ignored. - - % This should be exactly the same as the table in PyString_FromFormat. - - % One should just refer to the other. - - % The descriptions for %zd and %zu are wrong, but the truth is complicated - - % because not all compilers support the %z width modifier -- we fake it - - % when necessary via interpolating PY_FORMAT_SIZE_T. - - % Similar comments apply to the %ll width modifier and - - % PY_FORMAT_LONG_LONG. - - % %u, %lu, %zu should have "new in Python 2.5" blurbs. - - - - - - - - Format Characters - - Type - - Comment - - %% - - n/a - - The literal % character. - - %c - - int - - A single character, - represented as an C int. - - %d - - int - - Exactly equivalent to - printf("%d"). - - %u - - unsigned int - - Exactly equivalent to - printf("%u"). - - %ld - - long - - Exactly equivalent to - printf("%ld"). - - %lu - - unsigned long - - Exactly equivalent to - printf("%lu"). - - %lld - - long long - - Exactly equivalent to - printf("%lld"). - - %llu - - unsigned - long long - - Exactly equivalent to - printf("%llu"). - - %zd - - Py_ssize_t - - Exactly equivalent to - printf("%zd"). - - %zu - - size_t - - Exactly equivalent to - printf("%zu"). - - %i - - int - - Exactly equivalent to - printf("%i"). - - %x - - int - - Exactly equivalent to - printf("%x"). - - %s - - char* - - A null-terminated C character - array. - - %p - - void* - - The hex representation of a C - pointer. Mostly equivalent to - printf("%p") except that - it is guaranteed to start with - the literal 0x regardless - of what the platform's - printf yields. - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result string, and any extra arguments discarded. - - The "%lld" and "%llu" format specifiers are only available - when HAVE_LONG_LONG is defined. - - Support for "%lld" and "%llu" added. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyErr_SetFromErrnoWithFilename(space, type, filename): """Similar to PyErr_SetFromErrno(), with the additional behavior that if @@ -2844,30 +2376,6 @@ raise SystemError and return NULL.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, PyObject], rffi.INT_real, error=-1) -def PyModule_AddObject(space, module, name, value): - """Add an object to module as name. This is a convenience function which can - be used from the module's initialization function. This steals a reference to - value. Return -1 on error, 0 on success. - """ - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, lltype.Signed], rffi.INT_real, error=-1) -def PyModule_AddIntConstant(space, module, name, value): - """Add an integer constant to module as name. This convenience function can be - used from the module's initialization function. Return -1 on error, 0 on - success. - """ - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], rffi.INT_real, error=-1) -def PyModule_AddStringConstant(space, module, name, value): - """Add a string constant to module as name. This convenience function can be - used from the module's initialization function. The string value must be - null-terminated. Return -1 on error, 0 on success. - """ - raise NotImplementedError - @cpython_api([PyObjectP, PyObjectP], rffi.INT_real, error=-1) def PyNumber_Coerce(space, p1, p2): """ @@ -2919,40 +2427,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, rffi.VOIDPP, Py_ssize_t], rffi.INT_real, error=-1) -def PyObject_AsReadBuffer(space, obj, buffer, buffer_len): - """Returns a pointer to a read-only memory location containing arbitrary data. - The obj argument must support the single-segment readable buffer - interface. On success, returns 0, sets buffer to the memory location - and buffer_len to the buffer length. Returns -1 and sets a - TypeError on error. - - - - This function used an int * type for buffer_len. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyObject_CheckReadBuffer(space, o): - """Returns 1 if o supports the single-segment readable buffer interface. - Otherwise returns 0. - """ - raise NotImplementedError - - at cpython_api([PyObject, rffi.VOIDPP, Py_ssize_t], rffi.INT_real, error=-1) -def PyObject_AsWriteBuffer(space, obj, buffer, buffer_len): - """Returns a pointer to a writeable memory location. The obj argument must - support the single-segment, character buffer interface. On success, - returns 0, sets buffer to the memory location and buffer_len to the - buffer length. Returns -1 and sets a TypeError on error. - - - - This function used an int * type for buffer_len. This might - require changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyObject_DelAttrString(space, o, attr_name): """Delete attribute named attr_name, for object o. Returns -1 on failure. @@ -2978,51 +2452,6 @@ for PyObject_Str().""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP, ], PyObject) -def PyObject_CallFunction(space, callable, format, ): - """ - - - - Call a callable Python object callable, with a variable number of C arguments. - The C arguments are described using a Py_BuildValue() style format - string. The format may be NULL, indicating that no arguments are provided. - Returns the result of the call on success, or NULL on failure. This is the - equivalent of the Python expression apply(callable, args) or - callable(*args). Note that if you only pass PyObject * args, - PyObject_CallFunctionObjArgs() is a faster alternative.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP, ], PyObject) -def PyObject_CallMethod(space, o, method, format, ): - """Call the method named method of object o with a variable number of C - arguments. The C arguments are described by a Py_BuildValue() format - string that should produce a tuple. The format may be NULL, indicating that - no arguments are provided. Returns the result of the call on success, or NULL - on failure. This is the equivalent of the Python expression o.method(args). - Note that if you only pass PyObject * args, - PyObject_CallMethodObjArgs() is a faster alternative.""" - raise NotImplementedError - - at cpython_api([PyObject, ], PyObject) -def PyObject_CallFunctionObjArgs(space, callable, ): - """Call a callable Python object callable, with a variable number of - PyObject* arguments. The arguments are provided as a variable number - of parameters followed by NULL. Returns the result of the call on success, or - NULL on failure. - """ - raise NotImplementedError - - at cpython_api([PyObject, PyObject, ], PyObject) -def PyObject_CallMethodObjArgs(space, o, name, ): - """Calls a method of the object o, where the name of the method is given as a - Python string object in name. It is called with a variable number of - PyObject* arguments. The arguments are provided as a variable number - of parameters followed by NULL. Returns the result of the call on success, or - NULL on failure. - """ - raise NotImplementedError - @cpython_api([PyObject], lltype.Signed, error=-1) def PyObject_HashNotImplemented(space, o): """Set a TypeError indicating that type(o) is not hashable and return -1. @@ -3367,160 +2796,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, ], PyObject) -def PyString_FromFormat(space, format, ): - """Take a C printf()-style format string and a variable number of - arguments, calculate the size of the resulting Python string and return a string - with the values formatted into it. The variable arguments must be C types and - must correspond exactly to the format characters in the format string. The - following format characters are allowed: - - % This should be exactly the same as the table in PyErr_Format. - - % One should just refer to the other. - - % The descriptions for %zd and %zu are wrong, but the truth is complicated - - % because not all compilers support the %z width modifier -- we fake it - - % when necessary via interpolating PY_FORMAT_SIZE_T. - - % Similar comments apply to the %ll width modifier and - - % PY_FORMAT_LONG_LONG. - - % %u, %lu, %zu should have "new in Python 2.5" blurbs. - - - - - - - - Format Characters - - Type - - Comment - - %% - - n/a - - The literal % character. - - %c - - int - - A single character, - represented as an C int. - - %d - - int - - Exactly equivalent to - printf("%d"). - - %u - - unsigned int - - Exactly equivalent to - printf("%u"). - - %ld - - long - - Exactly equivalent to - printf("%ld"). - - %lu - - unsigned long - - Exactly equivalent to - printf("%lu"). - - %lld - - long long - - Exactly equivalent to - printf("%lld"). - - %llu - - unsigned - long long - - Exactly equivalent to - printf("%llu"). - - %zd - - Py_ssize_t - - Exactly equivalent to - printf("%zd"). - - %zu - - size_t - - Exactly equivalent to - printf("%zu"). - - %i - - int - - Exactly equivalent to - printf("%i"). - - %x - - int - - Exactly equivalent to - printf("%x"). - - %s - - char* - - A null-terminated C character - array. - - %p - - void* - - The hex representation of a C - pointer. Mostly equivalent to - printf("%p") except that - it is guaranteed to start with - the literal 0x regardless - of what the platform's - printf yields. - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result string, and any extra arguments discarded. - - The "%lld" and "%llu" format specifiers are only available - when HAVE_LONG_LONG is defined. - - Support for "%lld" and "%llu" added.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, va_list], PyObject) -def PyString_FromFormatV(space, format, vargs): - """Identical to PyString_FromFormat() except that it takes exactly two - arguments.""" - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyString_GET_SIZE(space, string): """Macro form of PyString_Size() but without error checking. @@ -3698,20 +2973,6 @@ """As above, but write to sys.stderr or stderr instead.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], lltype.Void) -def Py_FatalError(space, message): - """ - - - - Print a fatal error message and kill the process. No cleanup is performed. - This function should only be invoked when a condition is detected that would - make it dangerous to continue using the Python interpreter; e.g., when the - object administration appears to be corrupted. On Unix, the standard C library - function abort() is called which will attempt to produce a core - file.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], lltype.Void) def Py_Exit(space, status): """ @@ -3737,18 +2998,6 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([Py_ssize_t, ], PyObject) -def PyTuple_Pack(space, n, ): - """Return a new tuple object of size n, or NULL on failure. The tuple values - are initialized to the subsequent n C arguments pointing to Python objects. - PyTuple_Pack(2, a, b) is equivalent to Py_BuildValue("(OO)", a, b). - - - - This function used an int type for n. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], PyObject, borrowed=True) def PyTuple_GET_ITEM(space, p, pos): """Like PyTuple_GetItem(), but does no checking of its arguments. From afa at codespeak.net Wed Apr 28 19:50:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:50:05 +0200 (CEST) Subject: [pypy-svn] r74195 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428175005.3BD37282B9D@codespeak.net> Author: afa Date: Wed Apr 28 19:50:03 2010 New Revision: 74195 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: It's not really necessary to record NULL as a borrowed reference. This also fix a crash in PyDict_GetItem, which returns NULL without declaring a container Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 28 19:50:03 2010 @@ -442,7 +442,7 @@ retval = make_ref(space, result, borrowed=borrowed) else: retval = result - if borrowed: + if borrowed and retval: add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) From afa at codespeak.net Wed Apr 28 19:52:31 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:52:31 +0200 (CEST) Subject: [pypy-svn] r74196 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428175231.AA140282B9D@codespeak.net> Author: afa Date: Wed Apr 28 19:52:30 2010 New Revision: 74196 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py Log: _PyObject_GetDictPtr it's mostly a stub, but the caller I've seen is prepared for it to return NULL and call getattr/setattr in this case. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/object.py Wed Apr 28 19:52:30 2010 @@ -2,7 +2,7 @@ from pypy.module.cpyext.api import cpython_api, generic_cpy_call, CANNOT_FAIL,\ Py_ssize_t, PyVarObject, Py_TPFLAGS_HEAPTYPE,\ Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef from pypy.module.cpyext.state import State from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject @@ -63,6 +63,10 @@ the fields used by the tp_traverse handler become invalid.""" pass + at cpython_api([PyObject], PyObjectP, error=CANNOT_FAIL) +def _PyObject_GetDictPtr(space, op): + return lltype.nullptr(PyObjectP.TO) + @cpython_api([PyObject], rffi.INT_real, error=-1) def PyObject_IsTrue(space, w_obj): return space.is_true(w_obj) From afa at codespeak.net Wed Apr 28 19:58:18 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 19:58:18 +0200 (CEST) Subject: [pypy-svn] r74197 - pypy/branch/cpython-extension/pypy/module/cpyext/test Message-ID: <20100428175818.D3198282B9D@codespeak.net> Author: afa Date: Wed Apr 28 19:58:17 2010 New Revision: 74197 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Log: make_ref() does not support types with multiple bases, this crashes wxPython. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Wed Apr 28 19:58:17 2010 @@ -1,4 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref import py import sys @@ -100,3 +102,18 @@ assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() + +class TestTypes(BaseApiTest): + def test_multiple_inheritance(self, space, api): + py.test.skip("This bug causes a crash in wxPython") + w_class = space.appexec([], """(): + class A(object): + pass + class B(object): + pass + class C(A, B): + pass + return C + """) + ref = make_ref(space, w_class) + api.Py_DecRef(ref) From benjamin at codespeak.net Wed Apr 28 22:56:11 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 28 Apr 2010 22:56:11 +0200 (CEST) Subject: [pypy-svn] r74198 - pypy/trunk/pypy Message-ID: <20100428205611.2F354282B9D@codespeak.net> Author: benjamin Date: Wed Apr 28 22:56:09 2010 New Revision: 74198 Modified: pypy/trunk/pypy/conftest.py Log: remove old skip Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Wed Apr 28 22:56:09 2010 @@ -8,8 +8,6 @@ from pypy.tool.udir import udir from pypy.tool.autopath import pypydir -rootdir = py.path.local(__file__).dirpath() - # pytest settings pytest_plugins = "resultlog", rsyncdirs = ['.', '../lib-python', '../demo'] @@ -515,11 +513,6 @@ class Directory(py.test.collect.Directory): - def consider_dir(self, path): - if path == rootdir.join("lib", "ctypes", "test"): - py.test.skip("These are the original ctypes tests.\n" - "You can try to run them with 'pypy-c runtests.py'.") - return super(Directory, self).consider_dir(path) def recfilter(self, path): # disable recursion in symlinked subdirectories From hpk at codespeak.net Wed Apr 28 23:00:26 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 28 Apr 2010 23:00:26 +0200 (CEST) Subject: [pypy-svn] r74199 - pypy/branch/py12 Message-ID: <20100428210026.F2576282B9D@codespeak.net> Author: hpk Date: Wed Apr 28 23:00:25 2010 New Revision: 74199 Added: pypy/branch/py12/ - copied from r74198, pypy/trunk/ Log: branch for porting pypy to the upcoming py-1.2.2 release which i'd also like to refine such that pypy does not use internal or deprecated APIs anymore. From afa at codespeak.net Wed Apr 28 23:01:23 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 23:01:23 +0200 (CEST) Subject: [pypy-svn] r74200 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428210123.19692282B9D@codespeak.net> Author: afa Date: Wed Apr 28 23:01:21 2010 New Revision: 74200 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Log: Oops, fix test: add_borrowed_object must be called to reset "state.last_container" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Wed Apr 28 23:01:21 2010 @@ -442,7 +442,7 @@ retval = make_ref(space, result, borrowed=borrowed) else: retval = result - if borrowed and retval: + if borrowed: add_borrowed_object(space, retval) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/pyobject.py Wed Apr 28 23:01:21 2010 @@ -305,6 +305,8 @@ state = space.fromcache(State) container_ptr = state.last_container state.last_container = 0 + if not obj: + return if not container_ptr: raise NullPointerException if container_ptr == -1: From hpk at codespeak.net Wed Apr 28 23:02:06 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 28 Apr 2010 23:02:06 +0200 (CEST) Subject: [pypy-svn] r74201 - pypy/branch/py12/py Message-ID: <20100428210206.C0C13282B9D@codespeak.net> Author: hpk Date: Wed Apr 28 23:02:05 2010 New Revision: 74201 Removed: pypy/branch/py12/py/ Log: removing inlined py lib in order to use the one from the environment for now From afa at codespeak.net Wed Apr 28 23:05:40 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 23:05:40 +0200 (CEST) Subject: [pypy-svn] r74202 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428210540.3A9B5282B9D@codespeak.net> Author: afa Date: Wed Apr 28 23:05:38 2010 New Revision: 74202 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: Handle multiple inheritance in PyTypeObject the "solid base" is the one defined by the TypeDef. Let's see how wxPython behaves now... Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_typeobject.py Wed Apr 28 23:05:38 2010 @@ -105,7 +105,6 @@ class TestTypes(BaseApiTest): def test_multiple_inheritance(self, space, api): - py.test.skip("This bug causes a crash in wxPython") w_class = space.appexec([], """(): class A(object): pass Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 28 23:05:38 2010 @@ -34,7 +34,7 @@ from pypy.rlib.rstring import rsplit from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.module.__builtin__.abstractinst import abstract_issubclass_w - +from pypy.module.__builtin__.interp_classobj import W_ClassObject WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False @@ -474,16 +474,13 @@ # c_tp_print, c_tp_getattr, c_tp_setattr # XXX implement # c_tp_compare and the following fields (see http://docs.python.org/c-api/typeobj.html ) - bases_w = w_type.bases_w - assert len(bases_w) <= 1 - pto.c_tp_base = lltype.nullptr(PyTypeObject) - pto.c_tp_bases = lltype.nullptr(PyObject.TO) + w_base = best_base(space, w_type.bases_w) + pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base)) - if bases_w: - ref = make_ref(space, bases_w[0]) - pto.c_tp_base = rffi.cast(PyTypeObjectPtr, ref) + pto.c_tp_bases = lltype.nullptr(PyObject.TO) PyPyType_Ready(space, pto, w_type) + pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -500,6 +497,38 @@ def PyType_Ready(space, pto): return PyPyType_Ready(space, pto, None) +def solid_base(space, w_type): + typedef = w_type.instancetypedef + return space.gettypeobject(typedef) + +def best_base(space, bases_w): + if not bases_w: + return None + + w_winner = None + for w_candidate in bases_w: + if isinstance(w_candidate, W_ClassObject): + # old-style base + continue + assert isinstance(w_candidate, W_TypeObject) + w_candidate = solid_base(space, w_candidate) + if not w_winner: + w_winner = w_candidate + elif space.abstract_issubclass_w(w_winner, w_candidate): + pass + elif space.abstract_issubclass_w(w_candidate, w_winner): + w_winner = w_candidate + else: + raise OperationError( + space.w_TypeError, + space.wrap("multiple bases have instance lay-out conflict")) + if w_winner is None: + raise OperationError( + space.w_TypeError, + space.wrap("a new-style class can't have only classic bases")) + + return w_winner + def inherit_slots(space, pto, w_base): # XXX missing: nearly everything base_pyo = make_ref(space, w_base) From afa at codespeak.net Wed Apr 28 23:21:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 23:21:19 +0200 (CEST) Subject: [pypy-svn] r74203 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100428212119.71746282B9D@codespeak.net> Author: afa Date: Wed Apr 28 23:21:18 2010 New Revision: 74203 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Log: really fix *all* tests. The best base is not the best solid base, but the base which *has* the best solid base. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/typeobject.py Wed Apr 28 23:21:18 2010 @@ -506,28 +506,31 @@ return None w_winner = None - for w_candidate in bases_w: - if isinstance(w_candidate, W_ClassObject): + w_base = None + for w_base_i in bases_w: + if isinstance(w_base_i, W_ClassObject): # old-style base continue - assert isinstance(w_candidate, W_TypeObject) - w_candidate = solid_base(space, w_candidate) + assert isinstance(w_base_i, W_TypeObject) + w_candidate = solid_base(space, w_base_i) if not w_winner: w_winner = w_candidate + w_base = w_base_i elif space.abstract_issubclass_w(w_winner, w_candidate): pass elif space.abstract_issubclass_w(w_candidate, w_winner): w_winner = w_candidate + w_base = w_base_i else: raise OperationError( space.w_TypeError, space.wrap("multiple bases have instance lay-out conflict")) - if w_winner is None: + if w_base is None: raise OperationError( space.w_TypeError, space.wrap("a new-style class can't have only classic bases")) - return w_winner + return w_base def inherit_slots(space, pto, w_base): # XXX missing: nearly everything From afa at codespeak.net Wed Apr 28 23:50:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 28 Apr 2010 23:50:08 +0200 (CEST) Subject: [pypy-svn] r74205 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428215008.90819282B9D@codespeak.net> Author: afa Date: Wed Apr 28 23:50:06 2010 New Revision: 74205 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py Log: PyMethod_New Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Wed Apr 28 23:50:06 2010 @@ -33,6 +33,15 @@ from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyMethod_New(space, w_func, w_self, w_cls): + """Return a new method object, with func being any callable object; this is the + function that will be called when the method is called. If this method should + be bound to an instance, self should be the instance and class should be the + class of self, otherwise self should be NULL and class should be the + class which provides the unbound method.""" + return Method(space, w_func, w_self, w_cls) + @cpython_api([PyObject], PyObject, borrowed=True) def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Wed Apr 28 23:50:06 2010 @@ -2324,15 +2324,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyMethod_New(space, func, self, cls): - """Return a new method object, with func being any callable object; this is the - function that will be called when the method is called. If this method should - be bound to an instance, self should be the instance and class should be the - class of self, otherwise self should be NULL and class should be the - class which provides the unbound method..""" - raise NotImplementedError - @cpython_api([PyObject], PyObject, borrowed=True) def PyMethod_Self(space, meth): """Return the instance associated with the method meth if it is bound, otherwise Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_funcobject.py Wed Apr 28 23:50:06 2010 @@ -27,3 +27,8 @@ w_function = space.getattr(w_method, space.wrap("im_func")) assert space.is_w(api.PyMethod_Function(w_method), w_function) + + w_class = space.getattr(w_method, space.wrap("im_class")) + w_self = space.getattr(w_method, space.wrap("im_self")) + w_method2 = api.PyMethod_New(w_function, w_self, w_class) + assert space.eq_w(w_method, w_method2) From afa at codespeak.net Thu Apr 29 00:01:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 00:01:43 +0200 (CEST) Subject: [pypy-svn] r74206 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428220143.1A6E0282B9E@codespeak.net> Author: afa Date: Thu Apr 29 00:01:41 2010 New Revision: 74206 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Log: PyDict_Values, PyDict_Keys, PyDict_Items Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Thu Apr 29 00:01:41 2010 @@ -72,6 +72,24 @@ """Empty an existing dictionary of all key-value pairs.""" space.call_method(w_obj, "clear") + at cpython_api([PyObject], PyObject) +def PyDict_Keys(space, w_obj): + """Return a PyListObject containing all the keys from the dictionary, + as in the dictionary method dict.keys().""" + return space.call_method(w_obj, "keys") + + at cpython_api([PyObject], PyObject) +def PyDict_Values(space, w_obj): + """Return a PyListObject containing all the values from the + dictionary p, as in the dictionary method dict.values().""" + return space.call_method(w_obj, "values") + + at cpython_api([PyObject], PyObject) +def PyDict_Items(space, w_obj): + """Return a PyListObject containing all the items from the + dictionary, as in the dictionary method dict.items().""" + return space.call_method(w_obj, "items") + @cpython_api([PyObject, Py_ssize_t, PyObjectP, PyObjectP], rffi.INT_real, error=CANNOT_FAIL) def PyDict_Next(space, w_obj, ppos, pkey, pvalue): """Iterate over all key-value pairs in the dictionary p. The Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Thu Apr 29 00:01:41 2010 @@ -811,24 +811,6 @@ key. Return 0 on success or -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyDict_Items(space, p): - """Return a PyListObject containing all the items from the - dictionary, as in the dictionary method dict.items().""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyDict_Keys(space, p): - """Return a PyListObject containing all the keys from the dictionary, - as in the dictionary method dict.keys().""" - raise NotImplementedError - - at cpython_api([PyObject], PyObject) -def PyDict_Values(space, p): - """Return a PyListObject containing all the values from the - dictionary p, as in the dictionary method dict.values().""" - raise NotImplementedError - @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) def PyDict_Merge(space, a, b, override): """Iterate over mapping object b adding key-value pairs to dictionary a. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Thu Apr 29 00:01:41 2010 @@ -51,3 +51,12 @@ i = space.wrap(2) assert not api.PyDict_Check(i) assert not api.PyDict_CheckExact(i) + + def test_keys(self, space, api): + w_d = space.newdict() + space.setitem(w_d, space.wrap("a"), space.wrap("b")) + + assert space.eq_w(api.PyDict_Keys(w_d), space.wrap(["a"])) + assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"])) + assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")])) + From hpk at codespeak.net Thu Apr 29 00:11:02 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 00:11:02 +0200 (CEST) Subject: [pypy-svn] r74207 - pypy/branch/py12/pypy Message-ID: <20100428221102.39134282B9E@codespeak.net> Author: hpk Date: Thu Apr 29 00:11:00 2010 New Revision: 74207 Modified: pypy/branch/py12/pypy/conftest.py Log: strike unused helper Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Thu Apr 29 00:11:00 2010 @@ -291,14 +291,6 @@ # uh? not found anywhere, fall back (which might raise AttributeError) return getattr(x, name) -class LazyObjSpaceGetter(object): - def __get__(self, obj, cls=None): - space = gettestobjspace() - if cls: - cls.space = space - return space - - class AppError(Exception): def __init__(self, excinfo): From hpk at codespeak.net Thu Apr 29 00:14:23 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 00:14:23 +0200 (CEST) Subject: [pypy-svn] r74208 - pypy/branch/py12/pypy Message-ID: <20100428221423.02F06282B9E@codespeak.net> Author: hpk Date: Thu Apr 29 00:14:22 2010 New Revision: 74208 Modified: pypy/branch/py12/pypy/conftest.py Log: revert 74207 and rather consider going to sleep? Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Thu Apr 29 00:14:22 2010 @@ -291,6 +291,14 @@ # uh? not found anywhere, fall back (which might raise AttributeError) return getattr(x, name) +class LazyObjSpaceGetter(object): + def __get__(self, obj, cls=None): + space = gettestobjspace() + if cls: + cls.space = space + return space + + class AppError(Exception): def __init__(self, excinfo): From afa at codespeak.net Thu Apr 29 00:45:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 00:45:51 +0200 (CEST) Subject: [pypy-svn] r74209 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428224551.D6CC8282B9E@codespeak.net> Author: afa Date: Thu Apr 29 00:45:50 2010 New Revision: 74209 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Log: PyDict_Update, PyDict_Copy Modified: pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/dictobject.py Thu Apr 29 00:45:50 2010 @@ -73,6 +73,20 @@ space.call_method(w_obj, "clear") @cpython_api([PyObject], PyObject) +def PyDict_Copy(space, w_obj): + """Return a new dictionary that contains the same key-value pairs as p. + """ + return space.call_method(w_obj, "copy") + + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyDict_Update(space, w_obj, w_other): + """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in + Python. Return 0 on success or -1 if an exception was raised. + """ + space.call_method(w_obj, "update", w_other) + return 0 + + at cpython_api([PyObject], PyObject) def PyDict_Keys(space, w_obj): """Return a PyListObject containing all the keys from the dictionary, as in the dictionary method dict.keys().""" Modified: pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/stubs.py Thu Apr 29 00:45:50 2010 @@ -799,12 +799,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyDict_Copy(space, p): - """Return a new dictionary that contains the same key-value pairs as p. - """ - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyDict_DelItemString(space, p, key): """Remove the entry in dictionary p which has a key specified by the string @@ -822,13 +816,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyDict_Update(space, a, b): - """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in - Python. Return 0 on success or -1 if an exception was raised. - """ - raise NotImplementedError - @cpython_api([PyObject, PyObject, rffi.INT_real], rffi.INT_real, error=-1) def PyDict_MergeFromSeq2(space, a, seq2, override): """Update or merge into dictionary a, from the key-value pairs in seq2. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_dictobject.py Thu Apr 29 00:45:50 2010 @@ -60,3 +60,14 @@ assert space.eq_w(api.PyDict_Values(w_d), space.wrap(["b"])) assert space.eq_w(api.PyDict_Items(w_d), space.wrap([("a", "b")])) + def test_update(self, space, api): + w_d = space.newdict() + space.setitem(w_d, space.wrap("a"), space.wrap("b")) + + w_d2 = api.PyDict_Copy(w_d) + assert not space.is_w(w_d2, w_d) + space.setitem(w_d, space.wrap("c"), space.wrap("d")) + space.setitem(w_d2, space.wrap("e"), space.wrap("f")) + + api.PyDict_Update(w_d, w_d2) + assert space.unwrap(w_d) == dict(a='b', c='d', e='f') From exarkun at codespeak.net Thu Apr 29 00:52:56 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Thu, 29 Apr 2010 00:52:56 +0200 (CEST) Subject: [pypy-svn] r74210 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428225256.C0CEC282B9E@codespeak.net> Author: exarkun Date: Thu Apr 29 00:52:55 2010 New Revision: 74210 Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py pypy/branch/cpython-extension/pypy/module/cpyext/thread.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Log: Add PyThread_get_thread_ident Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 29 00:52:55 2010 @@ -38,6 +38,7 @@ state.non_heaptypes[:] = [] # import these modules to register api functions by side-effect +import pypy.module.cpyext.thread import pypy.module.cpyext.pyobject import pypy.module.cpyext.boolobject import pypy.module.cpyext.floatobject Added: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py Thu Apr 29 00:52:55 2010 @@ -0,0 +1,20 @@ + +import thread, threading + +from pypy.module.cpyext.test.test_api import BaseApiTest + + +class TestPyThread(BaseApiTest): + def test_get_thread_ident(self, space, api): + results = [] + def some_thread(): + w_res = api.PyThread_get_thread_ident(space) + results.append((space.int_w(w_res), thread.get_ident())) + + some_thread() + assert results[0][0] == results[0][1] + + th = threading.Thread(target=some_thread, args=()) + th.start() + th.join() + assert results[1][0] == results[1][1] Added: pypy/branch/cpython-extension/pypy/module/cpyext/thread.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/thread.py Thu Apr 29 00:52:55 2010 @@ -0,0 +1,8 @@ + +from pypy.module.thread import os_thread +from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api +from pypy.rpython.lltypesystem import rffi + + at cpython_api([], rffi.LONG, error=CANNOT_FAIL) +def PyThread_get_thread_ident(space): + return os_thread.get_ident(space) From exarkun at codespeak.net Thu Apr 29 01:41:31 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Thu, 29 Apr 2010 01:41:31 +0200 (CEST) Subject: [pypy-svn] r74211 - in pypy/branch/cpython-extension/pypy/module/cpyext: . test Message-ID: <20100428234131.C9D92282B9E@codespeak.net> Author: exarkun Date: Thu Apr 29 01:41:30 2010 New Revision: 74211 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py pypy/branch/cpython-extension/pypy/module/cpyext/thread.py Log: Avoid the unnecessary wrapping in get_thread_ident; try to implement acquire_lock, and fail. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_thread.py Thu Apr 29 01:41:30 2010 @@ -1,6 +1,7 @@ import thread, threading +from pypy.module.thread.ll_thread import allocate_ll_lock from pypy.module.cpyext.test.test_api import BaseApiTest @@ -8,8 +9,8 @@ def test_get_thread_ident(self, space, api): results = [] def some_thread(): - w_res = api.PyThread_get_thread_ident(space) - results.append((space.int_w(w_res), thread.get_ident())) + res = api.PyThread_get_thread_ident(space) + results.append((res, thread.get_ident())) some_thread() assert results[0][0] == results[0][1] @@ -18,3 +19,18 @@ th.start() th.join() assert results[1][0] == results[1][1] + + assert results[0][0] != results[1][0] + + + # def test_acquire_lock(self, space, api): + # lock = allocate_ll_lock() + # assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 + # assert api.PyThread_acquire_lock(lock, space.w_int(1)) == 0 + + + # def test_release_lock(self, space, api): + # lock = allocate_ll_lock() + # api.PyThread_acquire_lock(lock, space.w_int(0)) + # api.PyThread_release_lock(lock) + # assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/thread.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/thread.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/thread.py Thu Apr 29 01:41:30 2010 @@ -1,8 +1,13 @@ -from pypy.module.thread import os_thread +from pypy.module.thread import ll_thread from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api from pypy.rpython.lltypesystem import rffi @cpython_api([], rffi.LONG, error=CANNOT_FAIL) def PyThread_get_thread_ident(space): - return os_thread.get_ident(space) + return ll_thread.get_ident() + + +# @cpython_api([ll_thread.TLOCKP, rffi.INT], rffi.INT, error=CANNOT_FAIL) +# def PyThread_acquire_lock(space, lock, waitflag): +# return ll_thread.Lock(lock).acquire(waitflag) From fijal at codespeak.net Thu Apr 29 05:43:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 05:43:20 +0200 (CEST) Subject: [pypy-svn] r74212 - pypy/branch/cpython-extension/pypy Message-ID: <20100429034320.EC0AA282B9E@codespeak.net> Author: fijal Date: Thu Apr 29 05:43:19 2010 New Revision: 74212 Modified: pypy/branch/cpython-extension/pypy/conftest.py Log: I don't really want to care. Allow this to work if there is no py.test around Modified: pypy/branch/cpython-extension/pypy/conftest.py ============================================================================== --- pypy/branch/cpython-extension/pypy/conftest.py (original) +++ pypy/branch/cpython-extension/pypy/conftest.py Thu Apr 29 05:43:19 2010 @@ -61,7 +61,7 @@ try: return _SPACECACHE[key] except KeyError: - if option.runappdirect: + if getattr(option, 'runappdirect', None): if name not in (None, 'std'): myname = getattr(sys, 'pypy_objspaceclass', '') if not myname.lower().startswith(name): From fijal at codespeak.net Thu Apr 29 05:50:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 05:50:24 +0200 (CEST) Subject: [pypy-svn] r74213 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429035024.58CAD282B9E@codespeak.net> Author: fijal Date: Thu Apr 29 05:50:22 2010 New Revision: 74213 Added: pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py (contents, props changed) Log: An attempt to patch distutils so you can easily build & test extensions with py.py, first go. Added: pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py Thu Apr 29 05:50:22 2010 @@ -0,0 +1,41 @@ + +""" A distutils-patching tool that allows testing CPython extensions without +building pypy-c. + +Run python setup.py build in your project directory + +You can import resulting .so with py.py --allworingmodules +""" + +import sys, os +dn = os.path.dirname +rootdir = dn(dn(dn(dn(__file__)))) +sys.path.insert(0, rootdir) +pypydir = os.path.join(rootdir, 'pypy') +f = open(os.path.join(pypydir, '_interfaces', 'pyconfig.h'), "w") +f.write("\n") +f.close() +sys.path.insert(0, os.getcwd()) +from distutils import sysconfig + +from pypy.tool.udir import udir +from pypy.conftest import gettestobjspace +from pypy.module.cpyext.api import setup_library +space = gettestobjspace(usemodules=['cpyext', 'thread']) +setup_library(space) + +inc_paths = str(udir) + +def get_python_inc(plat_specific=0, prefix=None): + if plat_specific: + return os.path.join(pypydir, '_interfaces') + return os.path.join(os.path.dirname(__file__), 'include') + +def patch_distutils(): + sysconfig.get_python_inc = get_python_inc + +# unconditionally, since it's supposed to be run as PYTHONSTARTUP +patch_distutils() + +del sys.argv[0] +execfile(sys.argv[0]) From fijal at codespeak.net Thu Apr 29 06:51:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 06:51:21 +0200 (CEST) Subject: [pypy-svn] r74214 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429045121.865CC282B9E@codespeak.net> Author: fijal Date: Thu Apr 29 06:51:19 2010 New Revision: 74214 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py Log: Ugh. Call the part that renames stuff and not the other part Modified: pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/presetup.py Thu Apr 29 06:51:19 2010 @@ -11,30 +11,29 @@ dn = os.path.dirname rootdir = dn(dn(dn(dn(__file__)))) sys.path.insert(0, rootdir) +from pypy.tool.udir import udir pypydir = os.path.join(rootdir, 'pypy') -f = open(os.path.join(pypydir, '_interfaces', 'pyconfig.h'), "w") +f = open(os.path.join(str(udir), 'pyconfig.h'), "w") f.write("\n") f.close() sys.path.insert(0, os.getcwd()) from distutils import sysconfig -from pypy.tool.udir import udir from pypy.conftest import gettestobjspace -from pypy.module.cpyext.api import setup_library +from pypy.module.cpyext.api import build_bridge space = gettestobjspace(usemodules=['cpyext', 'thread']) -setup_library(space) +build_bridge(space) inc_paths = str(udir) def get_python_inc(plat_specific=0, prefix=None): if plat_specific: - return os.path.join(pypydir, '_interfaces') + return str(udir) return os.path.join(os.path.dirname(__file__), 'include') def patch_distutils(): sysconfig.get_python_inc = get_python_inc -# unconditionally, since it's supposed to be run as PYTHONSTARTUP patch_distutils() del sys.argv[0] From fijal at codespeak.net Thu Apr 29 06:56:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 06:56:40 +0200 (CEST) Subject: [pypy-svn] r74215 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429045640.CE0FE282B9E@codespeak.net> Author: fijal Date: Thu Apr 29 06:56:39 2010 New Revision: 74215 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py Log: Try printing the rpython-level error via infrastructure that we already have (and abort) Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 29 06:56:39 2010 @@ -380,6 +380,8 @@ return int(space.is_w(w_obj_type, w_type)) return check, check_exact +pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void) + # Make the wrapper for the cases (1) and (2) def make_wrapper(space, callable): names = callable.api_func.argnames @@ -448,11 +450,14 @@ retval = rffi.cast(callable.api_func.restype, result) except NullPointerException: print "Container not registered by %s" % callable.__name__ - except Exception: + except Exception, e: if not we_are_translated(): import traceback traceback.print_exc() - print "Fatal exception encountered" + print str(e) + # we can't do much here, since we're in ctypes, swallow + else: + pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 return retval callable._always_inline_ = True From hpk at codespeak.net Thu Apr 29 11:02:42 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 11:02:42 +0200 (CEST) Subject: [pypy-svn] r74216 - in pypy/branch/py12/pypy: jit/tl/test module/_sre/test tool tool/pytest tool/test translator/platform Message-ID: <20100429090242.2A7E4282BDA@codespeak.net> Author: hpk Date: Thu Apr 29 11:02:40 2010 New Revision: 74216 Modified: pypy/branch/py12/pypy/jit/tl/test/test_pypyjit.py pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py pypy/branch/py12/pypy/tool/ansi_mandelbrot.py pypy/branch/py12/pypy/tool/ansi_print.py pypy/branch/py12/pypy/tool/pytest/appsupport.py pypy/branch/py12/pypy/tool/test/test_pytestsupport.py pypy/branch/py12/pypy/translator/platform/__init__.py Log: use new public API of py.test (upcoming as 1.2.2) instead of internal imports Modified: pypy/branch/py12/pypy/jit/tl/test/test_pypyjit.py ============================================================================== --- pypy/branch/py12/pypy/jit/tl/test/test_pypyjit.py (original) +++ pypy/branch/py12/pypy/jit/tl/test/test_pypyjit.py Thu Apr 29 11:02:40 2010 @@ -21,7 +21,7 @@ def check_crasher(func_name): try: JIT_EXECUTABLE.sysexec(CRASH_FILE, func_name) - except py.impl.process.cmdexec.ExecutionFailed, e: + except py.process.cmdexec.Error, e: print "stderr" print "------" print e.err Modified: pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/py12/pypy/module/_sre/test/test_app_sre.py Thu Apr 29 11:02:40 2010 @@ -3,7 +3,6 @@ from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option -from py.impl.test.outcome import Skipped def init_globals_hack(space): space.appexec([space.wrap(autopath.this_dir)], """(this_dir): @@ -288,7 +287,7 @@ # This imports support_test_sre as the global "s" try: cls.space = gettestobjspace(usemodules=('_locale',)) - except Skipped: + except py.test.skip.Exception: cls.space = gettestobjspace(usemodules=('_rawffi',)) init_globals_hack(cls.space) @@ -548,7 +547,7 @@ def setup_class(cls): try: cls.space = gettestobjspace(usemodules=('_locale',)) - except Skipped: + except py.test.skip.Exception: cls.space = gettestobjspace(usemodules=('_rawffi',)) # This imports support_test_sre as the global "s" init_globals_hack(cls.space) Modified: pypy/branch/py12/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/branch/py12/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/branch/py12/pypy/tool/ansi_mandelbrot.py Thu Apr 29 11:02:40 2010 @@ -1,6 +1,6 @@ import sys -from py.impl.io.terminalwriter import ansi_print, get_terminal_width +from py.io import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/branch/py12/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/py12/pypy/tool/ansi_print.py (original) +++ pypy/branch/py12/pypy/tool/ansi_print.py Thu Apr 29 11:02:40 2010 @@ -3,8 +3,7 @@ """ import sys - -from py.impl.io.terminalwriter import ansi_print +from py.io import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: Modified: pypy/branch/py12/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/py12/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/py12/pypy/tool/pytest/appsupport.py Thu Apr 29 11:02:40 2010 @@ -1,10 +1,7 @@ import autopath import py -import py.impl.code.assertion -from py.impl.code import _assertionold as exprinfo from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from py.impl.test.outcome import ExceptionFailure # ____________________________________________________________ @@ -158,7 +155,7 @@ source = None from pypy import conftest if source and not conftest.option.nomagic: - msg = exprinfo.interpret(source, runner, should_fail=True) + msg = py.code._reinterpret_old(source, runner, should_fail=True) space.setattr(w_self, space.wrap('args'), space.newtuple([space.wrap(msg)])) w_msg = space.wrap(msg) @@ -254,7 +251,7 @@ if not value.match(space, w_ExpectedException): raise type, value, tb return excinfo - except ExceptionFailure, e: + except py.test.raises.Exception, e: e.tbindex = getattr(e, 'tbindex', -1) - 1 raise Modified: pypy/branch/py12/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/branch/py12/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/py12/pypy/tool/test/test_pytestsupport.py Thu Apr 29 11:02:40 2010 @@ -1,5 +1,4 @@ import autopath -from py.impl.code import _assertionold as exprinfo from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.argument import Arguments @@ -23,8 +22,8 @@ co = PyCode._from_code(space, somefunc.func_code) pyframe = PyFrame(space, co, space.newdict(), None) runner = AppFrame(space, pyframe) - exprinfo.run("f = lambda x: x+1", runner) - msg = exprinfo.interpret("assert isinstance(f(2), float)", runner) + py.code.reinterpret_old("f = lambda x: x+1", runner, should_fail=False) + msg = py.code.reinterpret_old("assert isinstance(f(2), float)", runner) assert msg.startswith("assert isinstance(3, float)\n" " + where 3 = ") Modified: pypy/branch/py12/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/__init__.py (original) +++ pypy/branch/py12/pypy/translator/platform/__init__.py Thu Apr 29 11:02:40 2010 @@ -6,7 +6,6 @@ import sys, py, os from pypy.tool.ansi_print import ansi_log -from py.impl.code.code import safe_repr log = py.log.Producer("platform") py.log.setconsumer("platform", ansi_log) @@ -19,9 +18,9 @@ def __repr__(self): if self.err: - return "" % safe_repr(self.err) + return "" % repr(self.err) else: - return "" % safe_repr(self.out) + return "" % repr(self.out) __str__ = __repr__ From hpk at codespeak.net Thu Apr 29 11:32:29 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 11:32:29 +0200 (CEST) Subject: [pypy-svn] r74217 - pypy/branch/py12/pypy/tool Message-ID: <20100429093229.01A70282BDA@codespeak.net> Author: hpk Date: Thu Apr 29 11:32:28 2010 New Revision: 74217 Removed: pypy/branch/py12/pypy/tool/statistic_over_time.py Log: remove quite outdated tool for pickling svn commit data the hard way - i don't think anybody used this for two years or so and there are probably much more efficient ways to do it these days. From hpk at codespeak.net Thu Apr 29 11:33:09 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 11:33:09 +0200 (CEST) Subject: [pypy-svn] r74218 - pypy/branch/py12/pypy/tool Message-ID: <20100429093309.1B7F2282BDA@codespeak.net> Author: hpk Date: Thu Apr 29 11:33:07 2010 New Revision: 74218 Modified: pypy/branch/py12/pypy/tool/genstatistic.py Log: fix tool that generates LOC statistics for pypy - not sure it's needed but it works so i leave it for now Modified: pypy/branch/py12/pypy/tool/genstatistic.py ============================================================================== --- pypy/branch/py12/pypy/tool/genstatistic.py (original) +++ pypy/branch/py12/pypy/tool/genstatistic.py Thu Apr 29 11:33:07 2010 @@ -1,7 +1,7 @@ import autopath import py -from py.impl.misc.cmdline import countloc +from py._cmdline import pycountloc as countloc from py.xml import raw pypydir = py.path.local(autopath.pypydir) From hpk at codespeak.net Thu Apr 29 11:39:46 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 11:39:46 +0200 (CEST) Subject: [pypy-svn] r74219 - pypy/branch/py12/pypy/lib/distributed Message-ID: <20100429093946.EE722282BDA@codespeak.net> Author: hpk Date: Thu Apr 29 11:39:45 2010 New Revision: 74219 Modified: pypy/branch/py12/pypy/lib/distributed/socklayer.py Log: leave a note that the "pygreen" dependency needs to be sorted - question is probably if 'pypy/lib/distributed' should just vanish for now. Modified: pypy/branch/py12/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/branch/py12/pypy/lib/distributed/socklayer.py (original) +++ pypy/branch/py12/pypy/lib/distributed/socklayer.py Thu Apr 29 11:39:45 2010 @@ -1,6 +1,8 @@ import py from socket import socket + +XXX needs import adaptation as 'green' is removed from py lib for years from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal From hpk at codespeak.net Thu Apr 29 14:20:51 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 14:20:51 +0200 (CEST) Subject: [pypy-svn] r74220 - in pypy/branch/py12/pypy/translator/platform: . test Message-ID: <20100429122051.E4FDA282BDA@codespeak.net> Author: hpk Date: Thu Apr 29 14:20:50 2010 New Revision: 74220 Modified: pypy/branch/py12/pypy/translator/platform/__init__.py pypy/branch/py12/pypy/translator/platform/test/test_platform.py Log: use new py.io.saferepr facility and add a test where it is used Modified: pypy/branch/py12/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/__init__.py (original) +++ pypy/branch/py12/pypy/translator/platform/__init__.py Thu Apr 29 14:20:50 2010 @@ -18,9 +18,9 @@ def __repr__(self): if self.err: - return "" % repr(self.err) + return "" % py.io.saferepr(self.err) else: - return "" % repr(self.out) + return "" % py.io.saferepr(self.out) __str__ = __repr__ Modified: pypy/branch/py12/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/branch/py12/pypy/translator/platform/test/test_platform.py (original) +++ pypy/branch/py12/pypy/translator/platform/test/test_platform.py Thu Apr 29 14:20:50 2010 @@ -5,6 +5,14 @@ from pypy.translator.platform import host from pypy.translator.tool.cbuild import ExternalCompilationInfo +def test_compilationerror_repr(): + # compilation error output/stdout may be large, make sure + # repr creates a limited version + c = CompilationError('', '*'*1000) + assert len(repr(c)) < 500 + c = CompilationError('*'*1000, '') + assert len(repr(c)) < 500 + class TestPlatform(object): platform = host strict_on_stderr = True From afa at codespeak.net Thu Apr 29 16:06:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 16:06:51 +0200 (CEST) Subject: [pypy-svn] r74221 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429140651.93E9D282BDD@codespeak.net> Author: afa Date: Thu Apr 29 16:06:49 2010 New Revision: 74221 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Log: Don't forget to register the container for borrowed references Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Thu Apr 29 16:06:49 2010 @@ -46,6 +46,7 @@ def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" assert isinstance(w_method, Method) + register_container(space, w_method) return w_method.w_function @cpython_api([PyObject], PyObject, borrowed=True) @@ -53,5 +54,6 @@ """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" assert isinstance(w_method, Method) + register_container(space, w_method) return w_method.w_class From afa at codespeak.net Thu Apr 29 16:07:35 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 16:07:35 +0200 (CEST) Subject: [pypy-svn] r74222 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429140735.CE51E282BDD@codespeak.net> Author: afa Date: Thu Apr 29 16:07:34 2010 New Revision: 74222 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Log: Fix import Modified: pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/funcobject.py Thu Apr 29 16:07:34 2010 @@ -2,7 +2,8 @@ from pypy.module.cpyext.api import ( PyObjectFields, generic_cpy_call, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref, Py_DecRef, make_typedescr +from pypy.module.cpyext.pyobject import ( + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, register_container) from pypy.interpreter.function import Function, Method PyFunctionObjectStruct = lltype.ForwardReference() From arigo at codespeak.net Thu Apr 29 16:08:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 16:08:22 +0200 (CEST) Subject: [pypy-svn] r74223 - in pypy/branch/blackhole-improvement/pypy/jit: backend backend/llgraph codewriter metainterp metainterp/test Message-ID: <20100429140822.B6E85282BDD@codespeak.net> Author: arigo Date: Thu Apr 29 16:08:20 2010 New Revision: 74223 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/backend/model.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Log: General progress. Also reimplement int_{add,sub,mul}_ovf. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Thu Apr 29 16:08:20 2010 @@ -263,17 +263,6 @@ # ---------- - def clear_exception(self): - llimpl.clear_exception() - - def get_overflow_error(self): - return (self.cast_adr_to_int(llimpl.get_overflow_error()), - llimpl.get_overflow_error_value()) - - def get_zero_division_error(self): - return (self.cast_adr_to_int(llimpl.get_zero_division_error()), - llimpl.get_zero_division_error_value()) - def sizeof(self, S): assert not isinstance(S, lltype.Ptr) return self.getdescr(symbolic.get_size(S)) @@ -329,42 +318,42 @@ # ---------- the backend-dependent operations ---------- - def do_arraylen_gc(self, arraybox, arraydescr): - array = arraybox.getref_base() - return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) - - def do_strlen(self, stringbox): - string = stringbox.getref_base() - return history.BoxInt(llimpl.do_strlen(0, string)) - - def do_strgetitem(self, stringbox, indexbox): - string = stringbox.getref_base() - index = indexbox.getint() - return history.BoxInt(llimpl.do_strgetitem(0, string, index)) - - def do_unicodelen(self, stringbox): - string = stringbox.getref_base() - return history.BoxInt(llimpl.do_unicodelen(0, string)) - - def do_unicodegetitem(self, stringbox, indexbox): - string = stringbox.getref_base() - index = indexbox.getint() - return history.BoxInt(llimpl.do_unicodegetitem(0, string, index)) - - def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): - assert isinstance(arraydescr, Descr) - array = arraybox.getref_base() - index = indexbox.getint() - if arraydescr.typeinfo == REF: - return history.BoxPtr(llimpl.do_getarrayitem_gc_ptr(array, index)) - elif arraydescr.typeinfo == INT: - return history.BoxInt(llimpl.do_getarrayitem_gc_int(array, index, - self.memo_cast)) - elif arraydescr.typeinfo == FLOAT: - return history.BoxFloat(llimpl.do_getarrayitem_gc_float(array, - index)) - else: - raise NotImplementedError +## def do_arraylen_gc(self, arraybox, arraydescr): +## array = arraybox.getref_base() +## return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) + +## def do_strlen(self, stringbox): +## string = stringbox.getref_base() +## return history.BoxInt(llimpl.do_strlen(0, string)) + +## def do_strgetitem(self, stringbox, indexbox): +## string = stringbox.getref_base() +## index = indexbox.getint() +## return history.BoxInt(llimpl.do_strgetitem(0, string, index)) + +## def do_unicodelen(self, stringbox): +## string = stringbox.getref_base() +## return history.BoxInt(llimpl.do_unicodelen(0, string)) + +## def do_unicodegetitem(self, stringbox, indexbox): +## string = stringbox.getref_base() +## index = indexbox.getint() +## return history.BoxInt(llimpl.do_unicodegetitem(0, string, index)) + +## def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): +## assert isinstance(arraydescr, Descr) +## array = arraybox.getref_base() +## index = indexbox.getint() +## if arraydescr.typeinfo == REF: +## return history.BoxPtr(llimpl.do_getarrayitem_gc_ptr(array, index)) +## elif arraydescr.typeinfo == INT: +## return history.BoxInt(llimpl.do_getarrayitem_gc_int(array, index, +## self.memo_cast)) +## elif arraydescr.typeinfo == FLOAT: +## return history.BoxFloat(llimpl.do_getarrayitem_gc_float(array, +## index)) +## else: +## raise NotImplementedError ## def do_getfield_gc(self, structbox, fielddescr): ## assert isinstance(fielddescr, Descr) @@ -424,9 +413,9 @@ assert isinstance(fielddescr, Descr) return llimpl.do_getfield_raw_float(struct, fielddescr.ofs) - def do_new(self, size): - assert isinstance(size, Descr) - return history.BoxPtr(llimpl.do_new(size.ofs)) +## def do_new(self, size): +## assert isinstance(size, Descr) +## return history.BoxPtr(llimpl.do_new(size.ofs)) def bh_new(self, sizedescr): assert isinstance(sizedescr, Descr) @@ -444,31 +433,31 @@ result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) return llmemory.cast_adr_to_int(result_adr) - def do_new_array(self, countbox, size): - assert isinstance(size, Descr) - count = countbox.getint() - return history.BoxPtr(llimpl.do_new_array(size.ofs, count)) +## def do_new_array(self, countbox, size): +## assert isinstance(size, Descr) +## count = countbox.getint() +## return history.BoxPtr(llimpl.do_new_array(size.ofs, count)) def bh_new_array(self, arraydescr, length): assert isinstance(arraydescr, Descr) return llimpl.do_new_array(arraydescr.ofs, length) - def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): - assert isinstance(arraydescr, Descr) - array = arraybox.getref_base() - index = indexbox.getint() - if arraydescr.typeinfo == REF: - newvalue = newvaluebox.getref_base() - llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - newvalue = newvaluebox.getint() - llimpl.do_setarrayitem_gc_int(array, index, newvalue, - self.memo_cast) - elif arraydescr.typeinfo == FLOAT: - newvalue = newvaluebox.getfloat() - llimpl.do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError +## def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): +## assert isinstance(arraydescr, Descr) +## array = arraybox.getref_base() +## index = indexbox.getint() +## if arraydescr.typeinfo == REF: +## newvalue = newvaluebox.getref_base() +## llimpl.do_setarrayitem_gc_ptr(array, index, newvalue) +## elif arraydescr.typeinfo == INT: +## newvalue = newvaluebox.getint() +## llimpl.do_setarrayitem_gc_int(array, index, newvalue, +## self.memo_cast) +## elif arraydescr.typeinfo == FLOAT: +## newvalue = newvaluebox.getfloat() +## llimpl.do_setarrayitem_gc_float(array, index, newvalue) +## else: +## raise NotImplementedError def bh_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): assert isinstance(arraydescr, Descr) @@ -532,28 +521,28 @@ assert isinstance(fielddescr, Descr) llimpl.do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - def do_same_as(self, box1): - return box1.clonebox() +## def do_same_as(self, box1): +## return box1.clonebox() - def do_newstr(self, lengthbox): - length = lengthbox.getint() - return history.BoxPtr(llimpl.do_newstr(0, length)) - - def do_newunicode(self, lengthbox): - length = lengthbox.getint() - return history.BoxPtr(llimpl.do_newunicode(0, length)) - - def do_strsetitem(self, stringbox, indexbox, newvaluebox): - string = stringbox.getref_base() - index = indexbox.getint() - newvalue = newvaluebox.getint() - llimpl.do_strsetitem(0, string, index, newvalue) - - def do_unicodesetitem(self, stringbox, indexbox, newvaluebox): - string = stringbox.getref_base() - index = indexbox.getint() - newvalue = newvaluebox.getint() - llimpl.do_unicodesetitem(0, string, index, newvalue) +## def do_newstr(self, lengthbox): +## length = lengthbox.getint() +## return history.BoxPtr(llimpl.do_newstr(0, length)) + +## def do_newunicode(self, lengthbox): +## length = lengthbox.getint() +## return history.BoxPtr(llimpl.do_newunicode(0, length)) + +## def do_strsetitem(self, stringbox, indexbox, newvaluebox): +## string = stringbox.getref_base() +## index = indexbox.getint() +## newvalue = newvaluebox.getint() +## llimpl.do_strsetitem(0, string, index, newvalue) + +## def do_unicodesetitem(self, stringbox, indexbox, newvaluebox): +## string = stringbox.getref_base() +## index = indexbox.getint() +## newvalue = newvaluebox.getint() +## llimpl.do_unicodesetitem(0, string, index, newvalue) def bh_call_i(self, func, calldescr, args_i, args_r, args_f): self._prepare_call(INT, calldescr, args_i, args_r, args_f) @@ -581,9 +570,9 @@ for x in args_f: llimpl.do_call_pushfloat(x) - def do_cast_ptr_to_int(self, ptrbox): - return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), - self.memo_cast)) +## def do_cast_ptr_to_int(self, ptrbox): +## return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), +## self.memo_cast)) def bh_cast_ptr_to_int(self, ptr): return llimpl.cast_to_int(ptr) @@ -596,7 +585,7 @@ return self.get_fail_descr_from_number(fail_index) -class OOtypeCPU(BaseCPU): +class OOtypeCPU_xxx_disabled(BaseCPU): is_oo = True ts = oohelper @@ -915,4 +904,4 @@ import pypy.jit.metainterp.executor pypy.jit.metainterp.executor.make_execute_list(LLtypeCPU) -pypy.jit.metainterp.executor.make_execute_list(OOtypeCPU) +##pypy.jit.metainterp.executor.make_execute_list(OOtypeCPU) Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/model.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/model.py Thu Apr 29 16:08:20 2010 @@ -155,32 +155,6 @@ # lltype specific operations # -------------------------- - def do_arraylen_gc(self, arraybox, arraydescr): - raise NotImplementedError - - def do_strlen(self, stringbox): - raise NotImplementedError - - def do_strgetitem(self, stringbox, indexbox): - raise NotImplementedError - - def do_unicodelen(self, stringbox): - raise NotImplementedError - - def do_unicodegetitem(self, stringbox, indexbox): - raise NotImplementedError - - def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): - raise NotImplementedError - - def do_arraycopy(self, calldescr, fnptr, sourcebox, destbox, - source_startbox, dest_startbox, lengthbox, arraydescr): - return self.do_call([fnptr, sourcebox, destbox, source_startbox, - dest_startbox, lengthbox], calldescr) - - def do_getfield_gc(self, structbox, fielddescr): - raise NotImplementedError - def bh_getfield_gc_i(self, struct, fielddescr): raise NotImplementedError def bh_getfield_gc_c(self, struct, fielddescr): @@ -192,9 +166,6 @@ def bh_getfield_gc_f(self, struct, fielddescr): raise NotImplementedError - def do_getfield_raw(self, structbox, fielddescr): - raise NotImplementedError - def bh_getfield_raw_i(self, struct, fielddescr): raise NotImplementedError def bh_getfield_raw_c(self, struct, fielddescr): @@ -206,28 +177,14 @@ def bh_getfield_raw_f(self, struct, fielddescr): raise NotImplementedError - def do_new(self, sizedescr): - raise NotImplementedError - def bh_new(self, sizedescr): raise NotImplementedError - - def do_new_with_vtable(self, classbox): - raise NotImplementedError - def bh_new_with_vtable(self, sizevtabledescr): raise NotImplementedError - - def bh_classof(self, struct): - raise NotImplementedError - - def do_new_array(self, lengthbox, arraydescr): - raise NotImplementedError - def bh_new_array(self, arraydescr, length): raise NotImplementedError - def do_setarrayitem_gc(self, arraybox, indexbox, newvaluebox, arraydescr): + def bh_classof(self, struct): raise NotImplementedError def bh_setarrayitem_gc_i(self, arraydescr, array, index, newvalue): @@ -237,12 +194,6 @@ def bh_setarrayitem_gc_f(self, arraydescr, array, index, newvalue): raise NotImplementedError - def do_setarrayitem_raw(self, arraybox, indexbox, newvaluebox, arraydescr): - raise NotImplementedError - - def do_setfield_gc(self, structbox, newvaluebox, fielddescr): - raise NotImplementedError - def bh_setfield_gc_i(self, struct, fielddescr, newvalue): raise NotImplementedError def bh_setfield_gc_c(self, struct, fielddescr, newvalue): @@ -254,9 +205,6 @@ def bh_setfield_gc_f(self, struct, fielddescr, newvalue): raise NotImplementedError - def do_setfield_raw(self, structbox, newvaluebox, fielddescr): - raise NotImplementedError - def bh_setfield_raw_i(self, struct, fielddescr, newvalue): raise NotImplementedError def bh_setfield_raw_c(self, struct, fielddescr, newvalue): @@ -268,19 +216,6 @@ def bh_setfield_raw_f(self, struct, fielddescr, newvalue): raise NotImplementedError - def do_newstr(self, lengthbox): - raise NotImplementedError - - def do_newunicode(self, lengthbox): - raise NotImplementedError - - def do_strsetitem(self, stringbox, indexbox, charbox): - raise NotImplementedError - - def do_unicodesetitem(self, stringbox, indexbox, charbox): - raise NotImplementedError - - # blackhole interface def bh_call_i(self, func, calldescr, args_i, args_r, args_f): raise NotImplementedError def bh_call_r(self, func, calldescr, args_i, args_r, args_f): @@ -290,41 +225,8 @@ def bh_call_v(self, func, calldescr, args_i, args_r, args_f): raise NotImplementedError - def do_call(self, args, calldescr): - raise NotImplementedError - - def do_call_assembler(self, args, token): - raise NotImplementedError - - def do_call_loopinvariant(self, args, calldescr): - return self.do_call(args, calldescr) - - def do_cond_call_gc_wb(self, args, calldescr): - raise NotImplementedError - - def do_cast_ptr_to_int(self, ptrbox): - raise NotImplementedError - def bh_cast_ptr_to_int(self, ptr): raise NotImplementedError - def do_call_may_force(self, args, calldescr): - return self.do_call(args, calldescr) - def force(self, force_token): raise NotImplementedError - - # ootype specific operations - # -------------------------- - - def do_runtimenew(self, classbox): - raise NotImplementedError - - def do_oosend(self, args, descr): - raise NotImplementedError - - def do_instanceof(self, instancebox, typedescr): - raise NotImplementedError - - def typedescr2classbox(self, descr): - raise NotImplementedError Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 29 16:08:20 2010 @@ -387,6 +387,9 @@ else: raise NoOp + def rewrite_op_bool_not(self, op): + return SpaceOperation('int_is_zero', op.args, op.result) + # ____________________________________________________________ def _with_prefix(prefix): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 29 16:08:20 2010 @@ -1,6 +1,7 @@ from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint +from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop @@ -28,6 +29,8 @@ value = intmask(value << (LONG_BIT-8)) >> (LONG_BIT-8) return value +NULL = lltype.nullptr(llmemory.GCREF.TO) + class BlackholeInterpBuilder(object): verbose = True @@ -76,6 +79,7 @@ def _get_method(self, name, argcodes): # def handler(self, code, position): + assert position >= 0 args = () next_argcode = 0 for argtype in argtypes: @@ -118,6 +122,7 @@ elif argtype == 'R': reg = self.registers_r[index] elif argtype == 'F': reg = self.registers_f[index] value.append(reg) + make_sure_not_resized(value) position += length elif argtype == 'self': value = self @@ -200,6 +205,7 @@ return BlackholeInterpreter(self) def release_interp(self, interp): + interp.cleanup_registers_r() self.blackholeinterps.append(interp) @@ -210,10 +216,11 @@ self.dispatch_loop = builder.dispatch_loop self.descrs = builder.descrs self.op_catch_exception = builder.op_catch_exception + self.cleanup_required_in_registers_r = -1 # if we_are_translated(): default_i = 0 - default_r = lltype.nullptr(llmemory.GCREF.TO) + default_r = NULL default_f = 0.0 else: default_i = MissingValue() @@ -237,6 +244,9 @@ self.copy_constants(self.registers_r, jitcode.constants_r) self.copy_constants(self.registers_f, jitcode.constants_f) code = jitcode.code + self.cleanup_required_in_registers_r = max( + self.cleanup_required_in_registers_r, + ord(code[-1])) while True: try: self.dispatch_loop(self, code, position) @@ -246,10 +256,34 @@ # ... except Exception, e: if not we_are_translated(): - if not isinstance(e, LLException): - raise + if isinstance(e, LLException): + pass # ok + elif isinstance(e, OverflowError): + e = self.get_standard_error_exception(OverflowError) + else: + raise # leave other exceptions be propagated position = self.handle_exception_in_frame(e, code) + def get_result_i(self): + return self.registers_i[0] + + def get_result_r(self): + return self.registers_r[0] + + def get_result_f(self): + return self.registers_f[0] + + def cleanup_registers_r(self): + # To avoid keeping references alive, this cleans up the registers_r. + # It does not clear the references set by copy_constants(), but + # these are all prebuilt constants anyway. + i = self.cleanup_required_in_registers_r + self.cleanup_required_in_registers_r = -1 + while i >= 0: + self.registers_r[i] = NULL + i -= 1 + self.exception_last_value = None + def handle_exception_in_frame(self, e, code): # This frame raises an exception. First try to see if # the exception is handled in the frame itself. @@ -265,6 +299,14 @@ target = ord(code[position+1]) | (ord(code[position+2])<<8) return target + def get_standard_error_exception(self, Class): + rtyper = self.cpu.rtyper + exdata = rtyper.getexceptiondata() + clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class) + evalue = exdata.get_standard_ll_exc_instance(rtyper, clsdef) + etype = rclass.ll_type(evalue) + return LLException(etype, evalue) + # XXX must be specialized # XXX the real performance impact of the following loop is unclear def copy_constants(self, registers, constants): @@ -282,15 +324,40 @@ @arguments("i", "i", returns="i") def opimpl_int_add(a, b): - return a + b + return intmask(a + b) @arguments("i", "i", returns="i") def opimpl_int_sub(a, b): - return a - b + return intmask(a - b) @arguments("i", "i", returns="i") def opimpl_int_mul(a, b): - return a * b + return intmask(a * b) + + @arguments("i", "i", returns="i") + def opimpl_int_add_ovf(a, b): + return ovfcheck(a + b) + + @arguments("i", "i", returns="i") + def opimpl_int_sub_ovf(a, b): + return ovfcheck(a - b) + + @arguments("i", "i", returns="i") + def opimpl_int_mul_ovf(a, b): + return ovfcheck(a * b) + + @arguments("i", "i", returns="i") + def opimpl_int_floordiv(a, b): + return llop.int_floordiv(lltype.Signed, a, b) + + @arguments("i", "i", returns="i") + def opimpl_uint_floordiv(a, b): + c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) + return intmask(c) + + @arguments("i", "i", returns="i") + def opimpl_int_mod(a, b): + return llop.int_mod(lltype.Signed, a, b) @arguments("i", "i", returns="i") def opimpl_int_and(a, b): @@ -305,69 +372,99 @@ return a ^ b @arguments("i", "i", returns="i") - def opimpl_int_floordiv(a, b): - return llop.int_floordiv(lltype.Signed, a, b) + def opimpl_int_rshift(a, b): + return a >> b @arguments("i", "i", returns="i") - def opimpl_int_mod(a, b): - return llop.int_mod(lltype.Signed, a, b) + def opimpl_int_lshift(a, b): + return intmask(a << b) @arguments("i", "i", returns="i") - def opimpl_uint_floordiv(a, b): - c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) + def opimpl_uint_rshift(a, b): + c = r_uint(a) >> r_uint(b) return intmask(c) + @arguments("i", returns="i") + def opimpl_int_neg(a): + return intmask(-a) + + @arguments("i", returns="i") + def opimpl_int_invert(a): + return intmask(~a) + @arguments("i", "i", returns="i") def opimpl_int_lt(a, b): - return int(a < b) + return intmask(a < b) @arguments("i", "i", returns="i") def opimpl_int_le(a, b): - return int(a <= b) + return intmask(a <= b) @arguments("i", "i", returns="i") def opimpl_int_eq(a, b): - return int(a == b) + return intmask(a == b) @arguments("i", "i", returns="i") def opimpl_int_ne(a, b): - return int(a != b) + return intmask(a != b) @arguments("i", "i", returns="i") def opimpl_int_gt(a, b): - return int(a > b) + return intmask(a > b) @arguments("i", "i", returns="i") def opimpl_int_ge(a, b): - return int(a >= b) + return intmask(a >= b) @arguments("i", returns="i") def opimpl_int_is_zero(a): - return int(not a) + return intmask(not a) @arguments("i", returns="i") def opimpl_int_is_true(a): - return int(bool(a)) + return intmask(bool(a)) + + @arguments("i", "i", returns="i") + def opimpl_uint_lt(a, b): + return intmask(r_uint(a) < r_uint(b)) + @arguments("i", "i", returns="i") + def opimpl_uint_le(a, b): + return intmask(r_uint(a) <= r_uint(b)) + @arguments("i", "i", returns="i") + def opimpl_uint_gt(a, b): + return intmask(r_uint(a) > r_uint(b)) + @arguments("i", "i", returns="i") + def opimpl_uint_ge(a, b): + return intmask(r_uint(a) >= r_uint(b)) @arguments("r", "r", returns="i") def opimpl_ptr_eq(a, b): - return int(a == b) + return intmask(a == b) @arguments("r", "r", returns="i") def opimpl_ptr_ne(a, b): - return int(a != b) + return intmask(a != b) @arguments("r", returns="i") def opimpl_ptr_iszero(a): - return int(not a) + return intmask(not a) @arguments("r", returns="i") def opimpl_ptr_nonzero(a): - return int(bool(a)) + return intmask(bool(a)) @arguments("self", "i") def opimpl_int_return(self, a): - self.result_i = a + self.registers_i[0] = a + raise LeaveFrame + @arguments("self", "r") + def opimpl_ref_return(self, a): + self.registers_r[0] = a + raise LeaveFrame + @arguments("self", "f") + def opimpl_float_return(self, a): + self.registers_f[0] = a + raise LeaveFrame + @arguments("self") + def opimpl_void_return(self): raise LeaveFrame @arguments("i", returns="i") def opimpl_int_copy(a): return a - @arguments("r", returns="r") def opimpl_ref_copy(a): return a - @arguments("f", returns="f") def opimpl_float_copy(a): return a Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 16:08:20 2010 @@ -12,129 +12,8 @@ from pypy.jit.metainterp.resoperation import rop -# Operations in the _ALWAYS_PURE part of the table of resoperation.py -# must return a ConstInt or ConstPtr. Other operations must return -# a BoxInt or BoxPtr or None. - # ____________________________________________________________ -def do_int_add(cpu, box1, box2): - return ConstInt(intmask(box1.getint() + box2.getint())) - -def do_int_sub(cpu, box1, box2): - return ConstInt(intmask(box1.getint() - box2.getint())) - -def do_int_mul(cpu, box1, box2): - return ConstInt(intmask(box1.getint() * box2.getint())) - -def do_int_floordiv(cpu, box1, box2): - z = llop.int_floordiv(lltype.Signed, box1.getint(), box2.getint()) - return ConstInt(z) - -def do_uint_floordiv(cpu, box1, box2): - z = llop.uint_floordiv(lltype.Unsigned, r_uint(box1.getint()), - r_uint(box2.getint())) - return ConstInt(intmask(z)) - -def do_int_mod(cpu, box1, box2): - z = llop.int_mod(lltype.Signed, box1.getint(), box2.getint()) - return ConstInt(z) - -def do_int_and(cpu, box1, box2): - return ConstInt(box1.getint() & box2.getint()) - -def do_int_or(cpu, box1, box2): - return ConstInt(box1.getint() | box2.getint()) - -def do_int_xor(cpu, box1, box2): - return ConstInt(box1.getint() ^ box2.getint()) - -def do_int_rshift(cpu, box1, box2): - return ConstInt(box1.getint() >> box2.getint()) - -def do_int_lshift(cpu, box1, box2): - return ConstInt(intmask(box1.getint() << box2.getint())) - -def do_uint_rshift(cpu, box1, box2): - v = r_uint(box1.getint()) >> r_uint(box2.getint()) - return ConstInt(intmask(v)) - -# ---------- - -def do_int_lt(cpu, box1, box2): - return ConstInt(box1.getint() < box2.getint()) - -def do_int_le(cpu, box1, box2): - return ConstInt(box1.getint() <= box2.getint()) - -def do_int_eq(cpu, box1, box2): - return ConstInt(box1.getint() == box2.getint()) - -def do_int_ne(cpu, box1, box2): - return ConstInt(box1.getint() != box2.getint()) - -def do_int_gt(cpu, box1, box2): - return ConstInt(box1.getint() > box2.getint()) - -def do_int_ge(cpu, box1, box2): - return ConstInt(box1.getint() >= box2.getint()) - -def do_uint_lt(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) < r_uint(box2.getint())) - -def do_uint_le(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) <= r_uint(box2.getint())) - -def do_uint_gt(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) > r_uint(box2.getint())) - -def do_uint_ge(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) >= r_uint(box2.getint())) - -# ---------- - -def do_int_is_true(cpu, box1): - return ConstInt(bool(box1.getint())) - -def do_int_neg(cpu, box1): - return ConstInt(intmask(-box1.getint())) - -def do_int_invert(cpu, box1): - return ConstInt(~box1.getint()) - -def do_bool_not(cpu, box1): - return ConstInt(not box1.getint()) - -def do_same_as(cpu, box1): - return box1 - -def do_oois(cpu, box1, box2): - tp = box1.type - assert tp == box2.type - if tp == INT: - x = box1.getint() == box2.getint() - elif tp == REF: - x = box1.getref_base() == box2.getref_base() - else: - assert False - return ConstInt(x) - -def do_ooisnot(cpu, box1, box2): - tp = box1.type - assert tp == box2.type - if tp == INT: - x = box1.getint() != box2.getint() - elif tp == REF: - x = box1.getref_base() != box2.getref_base() - else: - assert False - return ConstInt(x) - -def do_subclassof(cpu, box1, box2): - return ConstInt(cpu.ts.subclassOf(cpu, box1, box2)) - -# ---------- - def do_int_add_ovf(cpu, box1, box2): x = box1.getint() y = box2.getint() @@ -243,6 +122,7 @@ def make_execute_list(cpuclass): + return # XXX from pypy.jit.backend.model import AbstractCPU if 0: # enable this to trace calls to do_xxx def wrap(fn): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 16:08:20 2010 @@ -45,9 +45,9 @@ def __init__(self, metainterp): self.metainterp = metainterp - self.boxes_i = [None] * 256 - self.boxes_r = [None] * 256 - self.boxes_f = [None] * 256 + self.registers_i = [None] * 256 + self.registers_r = [None] * 256 + self.registers_f = [None] * 256 def setup(self, jitcode, greenkey=None): assert isinstance(jitcode, JitCode) @@ -57,25 +57,38 @@ # this is not None for frames that are recursive portal calls self.greenkey = greenkey # copy the constants in place - self.copy_constants(self.boxes_i, jitcode.constants_i, ConstInt) - self.copy_constants(self.boxes_r, jitcode.constants_r, ConstPtr) - self.copy_constants(self.boxes_f, jitcode.constants_f, ConstFloat) - - def copy_constants(self, boxes, constants, ConstClass): - """Copy jitcode.constants[0] to boxes[255], - jitcode.constants[1] to boxes[254], - jitcode.constants[2] to boxes[253], etc.""" + self.copy_constants(self.registers_i, jitcode.constants_i, ConstInt) + self.copy_constants(self.registers_r, jitcode.constants_r, ConstPtr) + self.copy_constants(self.registers_f, jitcode.constants_f, ConstFloat) + + def copy_constants(self, registers, constants, ConstClass): + """Copy jitcode.constants[0] to registers[255], + jitcode.constants[1] to registers[254], + jitcode.constants[2] to registers[253], etc.""" i = len(constants) - 1 while i >= 0: j = 255 - i assert j >= 0 - boxes[j] = ConstClass(constants[i]) + registers[j] = ConstClass(constants[i]) i -= 1 copy_constants._annspecialcase_ = 'specialize:arg(3)' # ------------------------------ # Decoding of the JitCode + def prepare_list_of_boxes(self, outvalue, startindex, position, argcode): + assert argcode in 'IRF' + code = self.bytecode + length = ord(code[position]) + position += 1 + for i in range(length): + index = ord(code[position+i]) + if argcode == 'I': reg = self.registers_i[index] + elif argcode == 'R': reg = self.registers_r[index] + elif argcode == 'F': reg = self.registers_f[index] + outvalue[startindex+i] = reg + prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' + def load_int(self): pc = self.pc result = ord(self.bytecode[pc]) @@ -188,7 +201,7 @@ @arguments("box") def opimpl_any_return(self, box): - return self.metainterp.finishframe(box) + self.metainterp.finishframe(box) opimpl_int_return = opimpl_any_return opimpl_ref_return = opimpl_any_return @@ -196,7 +209,7 @@ @arguments() def opimpl_void_return(self): - return self.metainterp.finishframe(None) + self.metainterp.finishframe(None) @arguments("jumptarget") def opimpl_goto(self, target): @@ -592,9 +605,9 @@ def opimpl_call(self, callee, varargs): return self.perform_call(callee, varargs) - @arguments("descr", "varargs") - def opimpl_residual_call(self, calldescr, varargs): - return self.do_residual_call(varargs, descr=calldescr, exc=True) + @arguments("box", "descr", "boxes2") + def opimpl_residual_call_ir_i(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) @arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): @@ -1008,9 +1021,9 @@ else: return self.metainterp.assert_no_exception() - def do_residual_call(self, argboxes, descr, exc): + def do_residual_call(self, funcbox, descr, argboxes, exc): effectinfo = descr.get_extra_info() - if effectinfo is None or effectinfo.forces_virtual_or_virtualizable: + if 0:# XXX effectinfo is None or effectinfo.forces_virtual_or_virtualizable: # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication @@ -1696,13 +1709,13 @@ count_i = count_r = count_f = 0 for box in original_boxes: if box.type == history.INT: - f.boxes_i[count_i] = box + f.registers_i[count_i] = box count_i += 1 elif box.type == history.REF: - f.boxes_r[count_r] = box + f.registers_r[count_r] = box count_r += 1 elif box.type == history.FLOAT: - f.boxes_f[count_f] = box + f.registers_f[count_f] = box count_f += 1 else: raise AssertionError(box.type) @@ -2021,25 +2034,65 @@ from pypy.jit.metainterp.blackhole import signedord # def handler(self, position): + assert position >= 0 args = () next_argcode = 0 code = self.bytecode position += 1 for argtype in argtypes: - if argtype == "box": + if argtype == "box": # a box, of whatever type argcode = argcodes[next_argcode] next_argcode = next_argcode + 1 if argcode == 'i': - value = self.boxes_i[ord(code[position])] + value = self.registers_i[ord(code[position])] elif argcode == 'c': value = ConstInt(signedord(code[position])) elif argcode == 'r': - value = self.boxes_r[ord(code[position])] + value = self.registers_r[ord(code[position])] elif argcode == 'f': - value = self.boxes_f[ord(code[position])] + value = self.registers_f[ord(code[position])] else: raise AssertionError("bad argcode") position += 1 + elif argtype == "descr": + assert argcodes[next_argcode] == 'd' + next_argcode = next_argcode + 1 + index = ord(code[position]) | (ord(code[position+1])<<8) + value = self.metainterp.staticdata.opcode_descrs[index] + position += 2 + elif argtype == "boxes": # a list of boxes of some type + length = ord(code[position]) + value = [None] * length + self.prepare_list_of_boxes(value, 0, position, + argcodes[next_argcode]) + next_argcode = next_argcode + 1 + position += 1 + length + elif argtype == "boxes2": # two lists of boxes merged into one + length1 = ord(code[position]) + position2 = position + 1 + length1 + length2 = ord(code[position2]) + value = [None] * (length1 + length2) + self.prepare_list_of_boxes(value, 0, position, + argcodes[next_argcode]) + self.prepare_list_of_boxes(value, length1, position2, + argcodes[next_argcode + 1]) + next_argcode = next_argcode + 2 + position = position2 + 1 + length2 + elif argtype == "boxes3": # three lists of boxes merged into one + length1 = ord(code[position]) + position2 = position + 1 + length1 + length2 = ord(code[position2]) + position3 = position2 + 1 + length2 + length3 = ord(code[position3]) + value = [None] * (length1 + length2 + length3) + self.prepare_list_of_boxes(value, 0, position, + argcodes[next_argcode]) + self.prepare_list_of_boxes(value, length1, position2, + argcodes[next_argcode + 1]) + self.prepare_list_of_boxes(value, length1 + length2, position3, + argcodes[next_argcode + 2]) + next_argcode = next_argcode + 3 + position = position3 + 1 + length3 else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) @@ -2058,13 +2111,13 @@ target_index = ord(code[position]) if result_argcode == 'i': assert resultbox.type == history.INT - self.boxes_i[target_index] = resultbox + self.registers_i[target_index] = resultbox elif result_argcode == 'r': assert resultbox.type == history.REF - self.boxes_r[target_index] = resultbox + self.registers_r[target_index] = resultbox elif result_argcode == 'f': assert resultbox.type == history.FLOAT - self.boxes_f[target_index] = resultbox + self.registers_f[target_index] = resultbox else: raise AssertionError("bad result argcode") # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resoperation.py Thu Apr 29 16:08:20 2010 @@ -177,10 +177,10 @@ 'FLOAT_GT/2b', 'FLOAT_GE/2b', # + 'INT_IS_ZERO/1b', 'INT_IS_TRUE/1b', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1b', # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu Apr 29 16:08:20 2010 @@ -43,7 +43,7 @@ else: raise TypeError(T) blackholeinterp.run(mainjitcode, 0) - return blackholeinterp.result_i + return blackholeinterp.get_result_i() def _run_with_pyjitpl(cw, mainjitcode, args, testself): from pypy.jit.metainterp import simple_optimize @@ -174,7 +174,7 @@ class OOJitMixin(JitMixin): type_system = 'ootype' - CPUClass = runner.OOtypeCPU + #CPUClass = runner.OOtypeCPU def setup_class(cls): py.test.skip("ootype tests skipped for now") @@ -538,6 +538,39 @@ res = self.interp_operations(f, [15]) assert res == -1 + def test_int_add_ovf(self): + def f(x, y): + try: + return ovfcheck(x + y) + except OverflowError: + return -42 + res = self.interp_operations(f, [-100, 2]) + assert res == -98 + res = self.interp_operations(f, [1, sys.maxint]) + assert res == -42 + + def test_int_sub_ovf(self): + def f(x, y): + try: + return ovfcheck(x - y) + except OverflowError: + return -42 + res = self.interp_operations(f, [-100, 2]) + assert res == -102 + res = self.interp_operations(f, [1, -sys.maxint]) + assert res == -42 + + def test_int_mul_ovf(self): + def f(x, y): + try: + return ovfcheck(x * y) + except OverflowError: + return -42 + res = self.interp_operations(f, [-100, 2]) + assert res == -200 + res = self.interp_operations(f, [-3, sys.maxint//2]) + assert res == -42 + def test_mod_ovf(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'y']) def f(n, x, y): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_blackhole.py Thu Apr 29 16:08:20 2010 @@ -37,7 +37,7 @@ blackholeinterp.setarg_i(0, 40) blackholeinterp.setarg_i(1, 2) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 42 + assert blackholeinterp.get_result_i() == 42 def test_simple_const(): jitcode = JitCode("test") @@ -48,7 +48,7 @@ 'int_return/i': 1}) blackholeinterp.setarg_i(1, 6) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 42 + assert blackholeinterp.get_result_i() == 42 def test_simple_bigconst(): jitcode = JitCode("test") @@ -59,7 +59,7 @@ 'int_return/i': 1}) blackholeinterp.setarg_i(1, 10000) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 42 + assert blackholeinterp.get_result_i() == 42 def test_simple_loop(): jitcode = JitCode("test") @@ -77,7 +77,7 @@ blackholeinterp.setarg_i(0x16, 6) # %i0 blackholeinterp.setarg_i(0x17, 100) # %i1 blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 100+6+5+4+3 + assert blackholeinterp.get_result_i() == 100+6+5+4+3 def test_simple_exception(): jitcode = JitCode("test") @@ -95,11 +95,11 @@ # blackholeinterp.setarg_i(0x9, 100) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 200 + assert blackholeinterp.get_result_i() == 200 # blackholeinterp.setarg_i(0x9, -100) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 42 + assert blackholeinterp.get_result_i() == 42 # ____________________________________________________________ From afa at codespeak.net Thu Apr 29 16:18:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 16:18:21 +0200 (CEST) Subject: [pypy-svn] r74224 - pypy/branch/cpython-extension/pypy/module/cpyext Message-ID: <20100429141821.B7889282BDC@codespeak.net> Author: afa Date: Thu Apr 29 16:18:20 2010 New Revision: 74224 Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO Log: Describe a problem about package imports. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/TODO ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/TODO (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/TODO Thu Apr 29 16:18:20 2010 @@ -26,3 +26,9 @@ - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. + + - Py_InitModule() does not seem to return the correct module when using dotted import. + for example: "import wx" imports "wx._core_", which calls Py_InitModule("_core_"). + This correcly creates and populates sys.modules["wx._core_"], but returns a new empty + module sys.modules["_core_"]. So functions go in the correct module, but types and + constants go into the wrong one. From arigo at codespeak.net Thu Apr 29 16:36:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 16:36:21 +0200 (CEST) Subject: [pypy-svn] r74225 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100429143621.F31DC282BDD@codespeak.net> Author: arigo Date: Thu Apr 29 16:36:20 2010 New Revision: 74225 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: More rewriting, and implement the float operations. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Thu Apr 29 16:36:20 2010 @@ -182,13 +182,14 @@ linkfalse, linktrue = block.exits if linkfalse.llexitcase == True: linkfalse, linktrue = linktrue, linkfalse + opname = 'goto_if_not' if isinstance(block.exitswitch, tuple): # special case produced by jitter.optimize_goto_if_not() - opname = 'goto_if_not_' + block.exitswitch[0] + if block.exitswitch[0] != 'int_is_true': + opname = 'goto_if_not_' + block.exitswitch[0] opargs = block.exitswitch[1:] else: assert block.exitswitch.concretetype == lltype.Bool - opname = 'goto_if_not' opargs = [block.exitswitch] # self.emitline(opname, TLabel(linkfalse), Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/jitter.py Thu Apr 29 16:36:20 2010 @@ -1,4 +1,4 @@ -import sys +import py, sys from pypy.rpython.lltypesystem import lltype, rstr from pypy.jit.metainterp.history import getkind from pypy.objspace.flow.model import SpaceOperation, Variable, Constant @@ -99,7 +99,8 @@ return False # variable is also used in cur block if v is op.result: if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', - 'int_gt', 'int_ge', 'int_is_true', + 'int_gt', 'int_ge', + 'int_is_zero', 'int_is_true', 'ptr_eq', 'ptr_ne', 'ptr_iszero', 'ptr_nonzero'): return False # not a supported operation @@ -314,7 +315,7 @@ op.args[0].concretetype.TO._hints.get('typeptr')) def handle_getfield_typeptr(self, op): - return SpaceOperation('classof', [op.args[0]], op.result) + return SpaceOperation('guard_class', [op.args[0]], op.result) def rewrite_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} @@ -387,8 +388,13 @@ else: raise NoOp - def rewrite_op_bool_not(self, op): - return SpaceOperation('int_is_zero', op.args, op.result) + for _old, _new in [('bool_not', 'int_is_zero'), + ('cast_bool_to_float', 'cast_int_to_float'), + ]: + exec py.code.Source(''' + def rewrite_op_%s(self, op): + return SpaceOperation(%r, op.args, op.result) + ''' % (_old, _new)).compile() # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Thu Apr 29 16:36:20 2010 @@ -34,4 +34,4 @@ blackholeinterp.setarg_i(0, 6) blackholeinterp.setarg_i(1, 100) blackholeinterp.run(jitcode, 0) - assert blackholeinterp.result_i == 100+6+5+4+3 + assert blackholeinterp.get_result_i() == 100+6+5+4+3 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Thu Apr 29 16:36:20 2010 @@ -332,8 +332,10 @@ def f(i): return not i + # note that 'goto_if_not_int_is_true' is actually the same thing + # as just 'goto_if_not'. self.encoding_test(f, [7], """ - goto_if_not_int_is_true L1, %i0 + goto_if_not L1, %i0 int_return $False L1: int_return $True Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_jitter.py Thu Apr 29 16:36:20 2010 @@ -180,7 +180,7 @@ v_result = varoftype(rclass.OBJECT.typeptr) op = SpaceOperation('getfield', [v_parent, c_name], v_result) op1 = Transformer(FakeCPU()).rewrite_operation(op) - assert op1.opname == 'classof' + assert op1.opname == 'guard_class' assert op1.args == [v_parent] assert op1.result == v_result Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 29 16:36:20 2010 @@ -162,7 +162,9 @@ # argcode should be 'i' too assert argcodes[next_argcode] == 'i' next_argcode = next_argcode + 1 - assert lltype.typeOf(result) == lltype.Signed + if lltype.typeOf(result) is lltype.Bool: + result = int(result) + assert lltype.typeOf(result) is lltype.Signed self.registers_i[ord(code[position])] = result position += 1 elif resulttype == 'r': @@ -176,7 +178,7 @@ # argcode should be 'f' too assert argcodes[next_argcode] == 'f' next_argcode = next_argcode + 1 - assert lltype.typeOf(result) == lltype.Float + assert lltype.typeOf(result) is lltype.Float self.registers_f[ord(code[position])] = result position += 1 elif resulttype == 'L': @@ -273,6 +275,14 @@ def get_result_f(self): return self.registers_f[0] + def _get_result_anytype(self): + "NOT_RPYTHON" + if self._return_type == 'int': return self.get_result_i() + if self._return_type == 'ref': return self.get_result_r() + if self._return_type == 'float': return self.get_result_f() + if self._return_type == 'void': return None + raise ValueError(self._return_type) + def cleanup_registers_r(self): # To avoid keeping references alive, this cleans up the registers_r. # It does not clear the references set by copy_constants(), but @@ -394,84 +404,160 @@ @arguments("i", "i", returns="i") def opimpl_int_lt(a, b): - return intmask(a < b) + return a < b @arguments("i", "i", returns="i") def opimpl_int_le(a, b): - return intmask(a <= b) + return a <= b @arguments("i", "i", returns="i") def opimpl_int_eq(a, b): - return intmask(a == b) + return a == b @arguments("i", "i", returns="i") def opimpl_int_ne(a, b): - return intmask(a != b) + return a != b @arguments("i", "i", returns="i") def opimpl_int_gt(a, b): - return intmask(a > b) + return a > b @arguments("i", "i", returns="i") def opimpl_int_ge(a, b): - return intmask(a >= b) + return a >= b @arguments("i", returns="i") def opimpl_int_is_zero(a): - return intmask(not a) + return not a @arguments("i", returns="i") def opimpl_int_is_true(a): - return intmask(bool(a)) + return bool(a) @arguments("i", "i", returns="i") def opimpl_uint_lt(a, b): - return intmask(r_uint(a) < r_uint(b)) + return r_uint(a) < r_uint(b) @arguments("i", "i", returns="i") def opimpl_uint_le(a, b): - return intmask(r_uint(a) <= r_uint(b)) + return r_uint(a) <= r_uint(b) @arguments("i", "i", returns="i") def opimpl_uint_gt(a, b): - return intmask(r_uint(a) > r_uint(b)) + return r_uint(a) > r_uint(b) @arguments("i", "i", returns="i") def opimpl_uint_ge(a, b): - return intmask(r_uint(a) >= r_uint(b)) + return r_uint(a) >= r_uint(b) @arguments("r", "r", returns="i") def opimpl_ptr_eq(a, b): - return intmask(a == b) + return a == b @arguments("r", "r", returns="i") def opimpl_ptr_ne(a, b): - return intmask(a != b) + return a != b @arguments("r", returns="i") def opimpl_ptr_iszero(a): - return intmask(not a) + return not a @arguments("r", returns="i") def opimpl_ptr_nonzero(a): - return intmask(bool(a)) + return bool(a) + + @arguments("i", returns="i") + def opimpl_int_copy(a): + return a + @arguments("r", returns="r") + def opimpl_ref_copy(a): + return a + @arguments("f", returns="f") + def opimpl_float_copy(a): + return a + + opimpl_int_guard_value = opimpl_int_copy + opimpl_ref_guard_value = opimpl_ref_copy + opimpl_float_guard_value = opimpl_float_copy + + # ---------- + # float operations + + @arguments("f", returns="f") + def opimpl_float_neg(a): + return -a + @arguments("f", returns="f") + def opimpl_float_abs(a): + return abs(a) + @arguments("f", returns="i") + def opimpl_float_is_true(a): + return bool(a) + + @arguments("f", "f", returns="f") + def opimpl_float_add(a, b): + return a + b + @arguments("f", "f", returns="f") + def opimpl_float_sub(a, b): + return a - b + @arguments("f", "f", returns="f") + def opimpl_float_mul(a, b): + return a * b + @arguments("f", "f", returns="f") + def opimpl_float_truediv(a, b): + return a / b + + @arguments("f", "f", returns="i") + def opimpl_float_lt(a, b): + return a < b + @arguments("f", "f", returns="i") + def opimpl_float_le(a, b): + return a <= b + @arguments("f", "f", returns="i") + def opimpl_float_eq(a, b): + return a == b + @arguments("f", "f", returns="i") + def opimpl_float_ne(a, b): + return a != b + @arguments("f", "f", returns="i") + def opimpl_float_gt(a, b): + return a > b + @arguments("f", "f", returns="i") + def opimpl_float_ge(a, b): + return a >= b + + @arguments("f", returns="i") + def opimpl_cast_float_to_int(a): + # note: we need to call int() twice to care for the fact that + # int(-2147483648.0) returns a long :-( + return int(int(a)) + + @arguments("i", returns="f") + def opimpl_cast_int_to_float(a): + return float(a) + + # ---------- + # control flow operations @arguments("self", "i") def opimpl_int_return(self, a): self.registers_i[0] = a + if not we_are_translated(): + self._return_type = "int" raise LeaveFrame + @arguments("self", "r") def opimpl_ref_return(self, a): self.registers_r[0] = a + if not we_are_translated(): + self._return_type = "ref" raise LeaveFrame + @arguments("self", "f") def opimpl_float_return(self, a): self.registers_f[0] = a + if not we_are_translated(): + self._return_type = "float" raise LeaveFrame + @arguments("self") def opimpl_void_return(self): + if not we_are_translated(): + self._return_type = "void" raise LeaveFrame - @arguments("i", returns="i") - def opimpl_int_copy(a): - return a - @arguments("r", returns="r") - def opimpl_ref_copy(a): - return a - @arguments("f", returns="f") - def opimpl_float_copy(a): - return a - - opimpl_int_guard_value = opimpl_int_copy - opimpl_ref_guard_value = opimpl_ref_copy - opimpl_float_guard_value = opimpl_float_copy + @arguments("L", "i", "pc", returns="L") + def opimpl_goto_if_not(target, a, pc): + if a: + return pc + else: + return target @arguments("L", "i", "i", "pc", returns="L") def opimpl_goto_if_not_int_lt(target, a, b, pc): @@ -522,13 +608,6 @@ else: return target - @arguments("L", "i", "pc", returns="L") - def opimpl_goto_if_not_int_is_true(target, a, pc): - if a: - return pc - else: - return target - @arguments("L", "r", "r", "pc", returns="L") def opimpl_goto_if_not_ptr_eq(target, a, b, pc): if a == b: @@ -740,7 +819,7 @@ return self.cpu.bh_new_with_vtable(descr) @arguments("self", "r", returns="i") - def opimpl_classof(self, struct): + def opimpl_guard_class(self, struct): return self.cpu.bh_classof(struct) @arguments("self", "r", returns="i") Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 16:36:20 2010 @@ -14,47 +14,6 @@ # ____________________________________________________________ -def do_int_add_ovf(cpu, box1, box2): - x = box1.getint() - y = box2.getint() - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - cpu._overflow_flag = ovf - return BoxInt(z) - -def do_int_sub_ovf(cpu, box1, box2): - x = box1.getint() - y = box2.getint() - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - cpu._overflow_flag = ovf - return BoxInt(z) - -def do_int_mul_ovf(cpu, box1, box2): - x = box1.getint() - y = box2.getint() - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - cpu._overflow_flag = ovf - return BoxInt(z) - -# ---------- - def do_float_neg(cpu, box1): return ConstFloat(-box1.getfloat()) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 16:36:20 2010 @@ -211,6 +211,10 @@ def opimpl_void_return(self): self.metainterp.finishframe(None) + @arguments("label") + def opimpl_catch_exception(self, target): + pass # see comment in blackhole.py:opimpl_catch_exception. + @arguments("jumptarget") def opimpl_goto(self, target): self.pc = target @@ -2060,6 +2064,11 @@ index = ord(code[position]) | (ord(code[position+1])<<8) value = self.metainterp.staticdata.opcode_descrs[index] position += 2 + elif argtype == "label": + assert argcodes[next_argcode] == 'L' + next_argcode = next_argcode + 1 + value = ord(code[position]) | (ord(code[position+1])<<8) + position += 2 elif argtype == "boxes": # a list of boxes of some type length = ord(code[position]) value = [None] * length Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Thu Apr 29 16:36:20 2010 @@ -43,7 +43,7 @@ else: raise TypeError(T) blackholeinterp.run(mainjitcode, 0) - return blackholeinterp.get_result_i() + return blackholeinterp._get_result_anytype() def _run_with_pyjitpl(cw, mainjitcode, args, testself): from pypy.jit.metainterp import simple_optimize From hpk at codespeak.net Thu Apr 29 16:55:58 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Apr 2010 16:55:58 +0200 (CEST) Subject: [pypy-svn] r74226 - in pypy/branch/py12: lib-python pypy Message-ID: <20100429145558.A115E282BDE@codespeak.net> Author: hpk Date: Thu Apr 29 16:55:57 2010 New Revision: 74226 Modified: pypy/branch/py12/lib-python/conftest.py pypy/branch/py12/pypy/conftest.py Log: avoid deprecation warnings by using new pytest collect related hooks Modified: pypy/branch/py12/lib-python/conftest.py ============================================================================== --- pypy/branch/py12/lib-python/conftest.py (original) +++ pypy/branch/py12/lib-python/conftest.py Thu Apr 29 16:55:57 2010 @@ -531,7 +531,8 @@ l.append(RunFileExternal(name, parent=self, regrtest=regrtest)) return l -Directory = RegrDirectory +def pytest_collect_directory(parent, path): + return RegrDirectory(path, parent) class RunFileExternal(py.test.collect.File): def __init__(self, name, parent, regrtest): Modified: pypy/branch/py12/pypy/conftest.py ============================================================================== --- pypy/branch/py12/pypy/conftest.py (original) +++ pypy/branch/py12/pypy/conftest.py Thu Apr 29 16:55:57 2010 @@ -196,7 +196,10 @@ if not hasattr(__builtin__, helper): setattr(__builtin__, helper, getattr(py.test, helper)) -class Module(py.test.collect.Module): +def pytest_pycollect_makemodule(path, parent): + return PyPyModule(path, parent) + +class PyPyModule(py.test.collect.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space at the class) ourselves. @@ -204,7 +207,7 @@ def __init__(self, *args, **kwargs): if hasattr(sys, 'pypy_objspaceclass'): option.conf_iocapture = "sys" # pypy cannot do FD-based - super(Module, self).__init__(*args, **kwargs) + super(PyPyModule, self).__init__(*args, **kwargs) def accept_regular_test(self): if option.runappdirect: @@ -235,7 +238,7 @@ def setup(self): # stick py.test raise in module globals -- carefully ensure_pytest_builtin_helpers() - super(Module, self).setup() + super(PyPyModule, self).setup() # if hasattr(mod, 'objspacename'): # mod.space = getttestobjspace(mod.objspacename) @@ -512,9 +515,5 @@ py.test.skip("pexpect not found") -class Directory(py.test.collect.Directory): - - def recfilter(self, path): - # disable recursion in symlinked subdirectories - return (py.test.collect.Directory.recfilter(self, path) - and path.check(link=0)) +def pytest_ignore_collect_path(path): + return path.check(link=1) From arigo at codespeak.net Thu Apr 29 17:00:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 17:00:43 +0200 (CEST) Subject: [pypy-svn] r74227 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100429150043.8BD59282BDE@codespeak.net> Author: arigo Date: Thu Apr 29 17:00:41 2010 New Revision: 74227 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Automatically port the simple functions from the BlackholeInterpreter to make them available to pyjitpl's boxed view of the world. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 29 17:00:41 2010 @@ -136,7 +136,7 @@ position += 2 else: raise AssertionError("bad argtype: %r" % (argtype,)) - args += (value,) + args = args + (value,) if verbose and not we_are_translated(): print '\t', name, list(args), Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 17:00:41 2010 @@ -6,83 +6,34 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, check_descr -from pypy.jit.metainterp.history import INT, REF, ConstFloat +from pypy.tool.sourcetools import func_with_new_name +from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop - # ____________________________________________________________ -def do_float_neg(cpu, box1): - return ConstFloat(-box1.getfloat()) - -def do_float_abs(cpu, box1): - return ConstFloat(abs(box1.getfloat())) - -def do_float_is_true(cpu, box1): - return ConstInt(bool(box1.getfloat())) - -def do_float_add(cpu, box1, box2): - return ConstFloat(box1.getfloat() + box2.getfloat()) - -def do_float_sub(cpu, box1, box2): - return ConstFloat(box1.getfloat() - box2.getfloat()) - -def do_float_mul(cpu, box1, box2): - return ConstFloat(box1.getfloat() * box2.getfloat()) - -def do_float_truediv(cpu, box1, box2): - return ConstFloat(box1.getfloat() / box2.getfloat()) - -def do_float_lt(cpu, box1, box2): - return ConstInt(box1.getfloat() < box2.getfloat()) - -def do_float_le(cpu, box1, box2): - return ConstInt(box1.getfloat() <= box2.getfloat()) - -def do_float_eq(cpu, box1, box2): - return ConstInt(box1.getfloat() == box2.getfloat()) - -def do_float_ne(cpu, box1, box2): - return ConstInt(box1.getfloat() != box2.getfloat()) - -def do_float_gt(cpu, box1, box2): - return ConstInt(box1.getfloat() > box2.getfloat()) +##def do_force_token(cpu): +## raise NotImplementedError -def do_float_ge(cpu, box1, box2): - return ConstInt(box1.getfloat() >= box2.getfloat()) +##def do_virtual_ref(cpu, box1, box2): +## raise NotImplementedError -def do_cast_float_to_int(cpu, box1): - # note: we need to call int() twice to care for the fact that - # int(-2147483648.0) returns a long :-( - return ConstInt(int(int(box1.getfloat()))) +##def do_virtual_ref_finish(cpu, box1, box2): +## raise NotImplementedError -def do_cast_int_to_float(cpu, box1): - return ConstFloat(float(box1.getint())) - -# ____________________________________________________________ - -def do_force_token(cpu): - raise NotImplementedError - -def do_virtual_ref(cpu, box1, box2): - raise NotImplementedError - -def do_virtual_ref_finish(cpu, box1, box2): - raise NotImplementedError - -def do_debug_merge_point(cpu, box1): - from pypy.jit.metainterp.warmspot import get_stats - loc = box1._get_str() - get_stats().add_merge_point_location(loc) +##def do_debug_merge_point(cpu, box1): +## from pypy.jit.metainterp.warmspot import get_stats +## loc = box1._get_str() +## get_stats().add_merge_point_location(loc) # ____________________________________________________________ def make_execute_list(cpuclass): - return # XXX from pypy.jit.backend.model import AbstractCPU + from pypy.jit.metainterp.blackhole import BlackholeInterpreter if 0: # enable this to trace calls to do_xxx def wrap(fn): def myfn(*args): @@ -115,15 +66,46 @@ raise Exception("duplicate entry for op number %d" % value) if key.endswith('_PURE'): key = key[:-5] - name = 'do_' + key.lower() - if hasattr(cpuclass, name): - execute[value] = wrap(getattr(cpuclass, name)) - elif name in globals(): - execute[value] = wrap(globals()[name]) - else: - assert hasattr(AbstractCPU, name), name + name = 'opimpl_' + key.lower() + if hasattr(BlackholeInterpreter, name): + func = make_execute_function_with_boxes( + key.lower(), + getattr(BlackholeInterpreter, name).im_func) + if func is not None: + execute[value] = func + continue + pass #XXX... cpuclass._execute_by_num_args = execute_by_num_args +def make_execute_function_with_boxes(name, func): + # Make a wrapper for 'func'. The func is a simple opimpl_xxx function + # from the BlackholeInterpreter class. The wrapper is a new function + # that receives and returns boxed values. + for argtype in func.argtypes: + if argtype not in ('i', 'r', 'f'): + return None + if func.resulttype not in ('i', 'r', 'f', None): + return None + argtypes = unrolling_iterable(func.argtypes) + resulttype = func.resulttype + # + def do(cpu, *argboxes): + newargs = () + for argtype in argtypes: + argbox = argboxes[0] + argboxes = argboxes[1:] + if argtype == 'i': value = argbox.getint() + elif argtype == 'r': value = argbox.getptr_base() + elif argtype == 'f': value = argbox.getfloat() + newargs = newargs + (value,) + # + result = func(*newargs) + # + if resulttype == 'i': return BoxInt(result) + if resulttype == 'r': return BoxPtr(result) + if resulttype == 'f': return BoxFloat(result) + # + return func_with_new_name(do, 'do_' + name) def get_execute_funclist(cpu, num_args): # workaround, similar to the next one Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 17:00:41 2010 @@ -196,7 +196,7 @@ exec py.code.Source(''' @arguments("box") def opimpl_%s(self, b): - self.execute(rop.%s, b) + return self.execute(rop.%s, b) ''' % (_opimpl, _opimpl.upper())).compile() @arguments("box") From arigo at codespeak.net Thu Apr 29 17:02:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 17:02:57 +0200 (CEST) Subject: [pypy-svn] r74228 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100429150257.46768282BDE@codespeak.net> Author: arigo Date: Thu Apr 29 17:02:55 2010 New Revision: 74228 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Log: Rename opimpl_xxx to bhimpl_xxx to avoid confusing them with the opimpl_xxx functions in pyjitpl. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 29 17:02:55 2010 @@ -141,7 +141,7 @@ if verbose and not we_are_translated(): print '\t', name, list(args), - # call the method opimpl_xxx() + # call the method bhimpl_xxx() try: result = unboundmethod(*args) except Exception, e: @@ -189,11 +189,11 @@ assert next_argcode == len(argcodes) return position # - # Get the opimpl_xxx method. If we get an AttributeError here, + # Get the bhimpl_xxx method. If we get an AttributeError here, # it means that either the implementation is missing, or that it # should not appear here at all but instead be transformed away # by codewriter/jitter.py. - unboundmethod = getattr(BlackholeInterpreter, 'opimpl_' + name).im_func + unboundmethod = getattr(BlackholeInterpreter, 'bhimpl_' + name).im_func verbose = self.verbose argtypes = unrolling_iterable(unboundmethod.argtypes) resulttype = unboundmethod.resulttype @@ -333,315 +333,315 @@ # ---------- @arguments("i", "i", returns="i") - def opimpl_int_add(a, b): + def bhimpl_int_add(a, b): return intmask(a + b) @arguments("i", "i", returns="i") - def opimpl_int_sub(a, b): + def bhimpl_int_sub(a, b): return intmask(a - b) @arguments("i", "i", returns="i") - def opimpl_int_mul(a, b): + def bhimpl_int_mul(a, b): return intmask(a * b) @arguments("i", "i", returns="i") - def opimpl_int_add_ovf(a, b): + def bhimpl_int_add_ovf(a, b): return ovfcheck(a + b) @arguments("i", "i", returns="i") - def opimpl_int_sub_ovf(a, b): + def bhimpl_int_sub_ovf(a, b): return ovfcheck(a - b) @arguments("i", "i", returns="i") - def opimpl_int_mul_ovf(a, b): + def bhimpl_int_mul_ovf(a, b): return ovfcheck(a * b) @arguments("i", "i", returns="i") - def opimpl_int_floordiv(a, b): + def bhimpl_int_floordiv(a, b): return llop.int_floordiv(lltype.Signed, a, b) @arguments("i", "i", returns="i") - def opimpl_uint_floordiv(a, b): + def bhimpl_uint_floordiv(a, b): c = llop.uint_floordiv(lltype.Unsigned, r_uint(a), r_uint(b)) return intmask(c) @arguments("i", "i", returns="i") - def opimpl_int_mod(a, b): + def bhimpl_int_mod(a, b): return llop.int_mod(lltype.Signed, a, b) @arguments("i", "i", returns="i") - def opimpl_int_and(a, b): + def bhimpl_int_and(a, b): return a & b @arguments("i", "i", returns="i") - def opimpl_int_or(a, b): + def bhimpl_int_or(a, b): return a | b @arguments("i", "i", returns="i") - def opimpl_int_xor(a, b): + def bhimpl_int_xor(a, b): return a ^ b @arguments("i", "i", returns="i") - def opimpl_int_rshift(a, b): + def bhimpl_int_rshift(a, b): return a >> b @arguments("i", "i", returns="i") - def opimpl_int_lshift(a, b): + def bhimpl_int_lshift(a, b): return intmask(a << b) @arguments("i", "i", returns="i") - def opimpl_uint_rshift(a, b): + def bhimpl_uint_rshift(a, b): c = r_uint(a) >> r_uint(b) return intmask(c) @arguments("i", returns="i") - def opimpl_int_neg(a): + def bhimpl_int_neg(a): return intmask(-a) @arguments("i", returns="i") - def opimpl_int_invert(a): + def bhimpl_int_invert(a): return intmask(~a) @arguments("i", "i", returns="i") - def opimpl_int_lt(a, b): + def bhimpl_int_lt(a, b): return a < b @arguments("i", "i", returns="i") - def opimpl_int_le(a, b): + def bhimpl_int_le(a, b): return a <= b @arguments("i", "i", returns="i") - def opimpl_int_eq(a, b): + def bhimpl_int_eq(a, b): return a == b @arguments("i", "i", returns="i") - def opimpl_int_ne(a, b): + def bhimpl_int_ne(a, b): return a != b @arguments("i", "i", returns="i") - def opimpl_int_gt(a, b): + def bhimpl_int_gt(a, b): return a > b @arguments("i", "i", returns="i") - def opimpl_int_ge(a, b): + def bhimpl_int_ge(a, b): return a >= b @arguments("i", returns="i") - def opimpl_int_is_zero(a): + def bhimpl_int_is_zero(a): return not a @arguments("i", returns="i") - def opimpl_int_is_true(a): + def bhimpl_int_is_true(a): return bool(a) @arguments("i", "i", returns="i") - def opimpl_uint_lt(a, b): + def bhimpl_uint_lt(a, b): return r_uint(a) < r_uint(b) @arguments("i", "i", returns="i") - def opimpl_uint_le(a, b): + def bhimpl_uint_le(a, b): return r_uint(a) <= r_uint(b) @arguments("i", "i", returns="i") - def opimpl_uint_gt(a, b): + def bhimpl_uint_gt(a, b): return r_uint(a) > r_uint(b) @arguments("i", "i", returns="i") - def opimpl_uint_ge(a, b): + def bhimpl_uint_ge(a, b): return r_uint(a) >= r_uint(b) @arguments("r", "r", returns="i") - def opimpl_ptr_eq(a, b): + def bhimpl_ptr_eq(a, b): return a == b @arguments("r", "r", returns="i") - def opimpl_ptr_ne(a, b): + def bhimpl_ptr_ne(a, b): return a != b @arguments("r", returns="i") - def opimpl_ptr_iszero(a): + def bhimpl_ptr_iszero(a): return not a @arguments("r", returns="i") - def opimpl_ptr_nonzero(a): + def bhimpl_ptr_nonzero(a): return bool(a) @arguments("i", returns="i") - def opimpl_int_copy(a): + def bhimpl_int_copy(a): return a @arguments("r", returns="r") - def opimpl_ref_copy(a): + def bhimpl_ref_copy(a): return a @arguments("f", returns="f") - def opimpl_float_copy(a): + def bhimpl_float_copy(a): return a - opimpl_int_guard_value = opimpl_int_copy - opimpl_ref_guard_value = opimpl_ref_copy - opimpl_float_guard_value = opimpl_float_copy + bhimpl_int_guard_value = bhimpl_int_copy + bhimpl_ref_guard_value = bhimpl_ref_copy + bhimpl_float_guard_value = bhimpl_float_copy # ---------- # float operations @arguments("f", returns="f") - def opimpl_float_neg(a): + def bhimpl_float_neg(a): return -a @arguments("f", returns="f") - def opimpl_float_abs(a): + def bhimpl_float_abs(a): return abs(a) @arguments("f", returns="i") - def opimpl_float_is_true(a): + def bhimpl_float_is_true(a): return bool(a) @arguments("f", "f", returns="f") - def opimpl_float_add(a, b): + def bhimpl_float_add(a, b): return a + b @arguments("f", "f", returns="f") - def opimpl_float_sub(a, b): + def bhimpl_float_sub(a, b): return a - b @arguments("f", "f", returns="f") - def opimpl_float_mul(a, b): + def bhimpl_float_mul(a, b): return a * b @arguments("f", "f", returns="f") - def opimpl_float_truediv(a, b): + def bhimpl_float_truediv(a, b): return a / b @arguments("f", "f", returns="i") - def opimpl_float_lt(a, b): + def bhimpl_float_lt(a, b): return a < b @arguments("f", "f", returns="i") - def opimpl_float_le(a, b): + def bhimpl_float_le(a, b): return a <= b @arguments("f", "f", returns="i") - def opimpl_float_eq(a, b): + def bhimpl_float_eq(a, b): return a == b @arguments("f", "f", returns="i") - def opimpl_float_ne(a, b): + def bhimpl_float_ne(a, b): return a != b @arguments("f", "f", returns="i") - def opimpl_float_gt(a, b): + def bhimpl_float_gt(a, b): return a > b @arguments("f", "f", returns="i") - def opimpl_float_ge(a, b): + def bhimpl_float_ge(a, b): return a >= b @arguments("f", returns="i") - def opimpl_cast_float_to_int(a): + def bhimpl_cast_float_to_int(a): # note: we need to call int() twice to care for the fact that # int(-2147483648.0) returns a long :-( return int(int(a)) @arguments("i", returns="f") - def opimpl_cast_int_to_float(a): + def bhimpl_cast_int_to_float(a): return float(a) # ---------- # control flow operations @arguments("self", "i") - def opimpl_int_return(self, a): + def bhimpl_int_return(self, a): self.registers_i[0] = a if not we_are_translated(): self._return_type = "int" raise LeaveFrame @arguments("self", "r") - def opimpl_ref_return(self, a): + def bhimpl_ref_return(self, a): self.registers_r[0] = a if not we_are_translated(): self._return_type = "ref" raise LeaveFrame @arguments("self", "f") - def opimpl_float_return(self, a): + def bhimpl_float_return(self, a): self.registers_f[0] = a if not we_are_translated(): self._return_type = "float" raise LeaveFrame @arguments("self") - def opimpl_void_return(self): + def bhimpl_void_return(self): if not we_are_translated(): self._return_type = "void" raise LeaveFrame @arguments("L", "i", "pc", returns="L") - def opimpl_goto_if_not(target, a, pc): + def bhimpl_goto_if_not(target, a, pc): if a: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_lt(target, a, b, pc): + def bhimpl_goto_if_not_int_lt(target, a, b, pc): if a < b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_le(target, a, b, pc): + def bhimpl_goto_if_not_int_le(target, a, b, pc): if a <= b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_eq(target, a, b, pc): + def bhimpl_goto_if_not_int_eq(target, a, b, pc): if a == b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_ne(target, a, b, pc): + def bhimpl_goto_if_not_int_ne(target, a, b, pc): if a != b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_gt(target, a, b, pc): + def bhimpl_goto_if_not_int_gt(target, a, b, pc): if a > b: return pc else: return target @arguments("L", "i", "i", "pc", returns="L") - def opimpl_goto_if_not_int_ge(target, a, b, pc): + def bhimpl_goto_if_not_int_ge(target, a, b, pc): if a >= b: return pc else: return target @arguments("L", "i", "pc", returns="L") - def opimpl_goto_if_not_int_is_zero(target, a, pc): + def bhimpl_goto_if_not_int_is_zero(target, a, pc): if not a: return pc else: return target @arguments("L", "r", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_eq(target, a, b, pc): + def bhimpl_goto_if_not_ptr_eq(target, a, b, pc): if a == b: return pc else: return target @arguments("L", "r", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_ne(target, a, b, pc): + def bhimpl_goto_if_not_ptr_ne(target, a, b, pc): if a != b: return pc else: return target @arguments("L", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_iszero(target, a, pc): + def bhimpl_goto_if_not_ptr_iszero(target, a, pc): if not a: return pc else: return target @arguments("L", "r", "pc", returns="L") - def opimpl_goto_if_not_ptr_nonzero(target, a, pc): + def bhimpl_goto_if_not_ptr_nonzero(target, a, pc): if a: return pc else: return target @arguments("L", returns="L") - def opimpl_goto(target): + def bhimpl_goto(target): return target @arguments("i", "d", "pc", returns="L") - def opimpl_switch(switchvalue, switchdict, pc): + def bhimpl_switch(switchvalue, switchdict, pc): assert isinstance(switchdict, SwitchDictDescr) try: return switchdict.dict[switchvalue] @@ -649,14 +649,14 @@ return pc @arguments("L") - def opimpl_catch_exception(target): + def bhimpl_catch_exception(target): """This is a no-op when run normally. When an exception occurs and the instruction that raised is immediately followed by a catch_exception, then the code in handle_exception_in_frame() will capture the exception and jump to 'target'.""" @arguments("self", "i", "L", "pc", returns="L") - def opimpl_goto_if_exception_mismatch(self, vtable, target, pc): + def bhimpl_goto_if_exception_mismatch(self, vtable, target, pc): adr = llmemory.cast_int_to_adr(vtable) bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) real_instance = self.exception_last_value @@ -667,20 +667,20 @@ return target @arguments("self", returns="i") - def opimpl_last_exception(self): + def bhimpl_last_exception(self): real_instance = self.exception_last_value assert real_instance adr = llmemory.cast_ptr_to_adr(real_instance.typeptr) return llmemory.cast_adr_to_int(adr) @arguments("self", returns="r") - def opimpl_last_exc_value(self): + def bhimpl_last_exc_value(self): real_instance = self.exception_last_value assert real_instance return lltype.cast_opaque_ptr(llmemory.GCREF, real_instance) @arguments("self") - def opimpl_reraise(self): + def bhimpl_reraise(self): real_instance = self.exception_last_value assert real_instance raise real_instance @@ -689,139 +689,139 @@ # the following operations are directly implemented by the backend @arguments("self", "i", "d", "R", returns="i") - def opimpl_residual_call_r_i(self, func, calldescr, args_r): + def bhimpl_residual_call_r_i(self, func, calldescr, args_r): return self.cpu.bh_call_i(func, calldescr, None, args_r, None) @arguments("self", "i", "d", "R", returns="r") - def opimpl_residual_call_r_r(self, func, calldescr, args_r): + def bhimpl_residual_call_r_r(self, func, calldescr, args_r): return self.cpu.bh_call_r(func, calldescr, None, args_r, None) @arguments("self", "i", "d", "R", returns="f") - def opimpl_residual_call_r_f(self, func, calldescr, args_r): + def bhimpl_residual_call_r_f(self, func, calldescr, args_r): return self.cpu.bh_call_f(func, calldescr, None, args_r, None) @arguments("self", "i", "d", "R") - def opimpl_residual_call_r_v(self, func, calldescr, args_r): + def bhimpl_residual_call_r_v(self, func, calldescr, args_r): self.cpu.bh_call_v(func, calldescr, None, args_r, None) @arguments("self", "i", "d", "I", "R", returns="i") - def opimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): + def bhimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_i(func, calldescr, args_i, args_r, None) @arguments("self", "i", "d", "I", "R", returns="r") - def opimpl_residual_call_ir_r(self, func, calldescr, args_i, args_r): + def bhimpl_residual_call_ir_r(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_r(func, calldescr, args_i, args_r, None) @arguments("self", "i", "d", "I", "R", returns="f") - def opimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): + def bhimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, None) @arguments("self", "i", "d", "I", "R") - def opimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): + def bhimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) @arguments("self", "i", "d", "I", "R", "F", returns="i") - def opimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): + def bhimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_i(func, calldescr, args_i, args_r, args_f) @arguments("self", "i", "d", "I", "R", "F", returns="r") - def opimpl_residual_call_irf_r(self, func, calldescr,args_i,args_r,args_f): + def bhimpl_residual_call_irf_r(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_r(func, calldescr, args_i, args_r, args_f) @arguments("self", "i", "d", "I", "R", "F", returns="f") - def opimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): + def bhimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): return self.cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) @arguments("self", "i", "d", "I", "R", "F") - def opimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): + def bhimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) @arguments("self", "d", "i", returns="r") - def opimpl_new_array(self, arraydescr, length): + def bhimpl_new_array(self, arraydescr, length): return self.cpu.bh_new_array(arraydescr, length) @arguments("self", "d", "r", "i", "r") - def opimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): + def bhimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) @arguments("self", "r", "d", returns="i") - def opimpl_getfield_gc_i(self, struct, fielddescr): + def bhimpl_getfield_gc_i(self, struct, fielddescr): return self.cpu.bh_getfield_gc_i(struct, fielddescr) @arguments("self", "r", "d", returns="i") - def opimpl_getfield_gc_c(self, struct, fielddescr): + def bhimpl_getfield_gc_c(self, struct, fielddescr): return self.cpu.bh_getfield_gc_c(struct, fielddescr) @arguments("self", "r", "d", returns="i") - def opimpl_getfield_gc_u(self, struct, fielddescr): + def bhimpl_getfield_gc_u(self, struct, fielddescr): return self.cpu.bh_getfield_gc_u(struct, fielddescr) @arguments("self", "r", "d", returns="r") - def opimpl_getfield_gc_r(self, struct, fielddescr): + def bhimpl_getfield_gc_r(self, struct, fielddescr): return self.cpu.bh_getfield_gc_r(struct, fielddescr) @arguments("self", "r", "d", returns="f") - def opimpl_getfield_gc_f(self, struct, fielddescr): + def bhimpl_getfield_gc_f(self, struct, fielddescr): return self.cpu.bh_getfield_gc_f(struct, fielddescr) - opimpl_getfield_gc_i_pure = opimpl_getfield_gc_i - opimpl_getfield_gc_c_pure = opimpl_getfield_gc_c - opimpl_getfield_gc_u_pure = opimpl_getfield_gc_u - opimpl_getfield_gc_r_pure = opimpl_getfield_gc_r - opimpl_getfield_gc_f_pure = opimpl_getfield_gc_f + bhimpl_getfield_gc_i_pure = bhimpl_getfield_gc_i + bhimpl_getfield_gc_c_pure = bhimpl_getfield_gc_c + bhimpl_getfield_gc_u_pure = bhimpl_getfield_gc_u + bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r + bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f @arguments("self", "i", "d", returns="i") - def opimpl_getfield_raw_i(self, struct, fielddescr): + def bhimpl_getfield_raw_i(self, struct, fielddescr): return self.cpu.bh_getfield_raw_i(struct, fielddescr) @arguments("self", "i", "d", returns="i") - def opimpl_getfield_raw_c(self, struct, fielddescr): + def bhimpl_getfield_raw_c(self, struct, fielddescr): return self.cpu.bh_getfield_raw_c(struct, fielddescr) @arguments("self", "i", "d", returns="i") - def opimpl_getfield_raw_u(self, struct, fielddescr): + def bhimpl_getfield_raw_u(self, struct, fielddescr): return self.cpu.bh_getfield_raw_u(struct, fielddescr) @arguments("self", "i", "d", returns="r") - def opimpl_getfield_raw_r(self, struct, fielddescr): + def bhimpl_getfield_raw_r(self, struct, fielddescr): return self.cpu.bh_getfield_raw_r(struct, fielddescr) @arguments("self", "i", "d", returns="f") - def opimpl_getfield_raw_f(self, struct, fielddescr): + def bhimpl_getfield_raw_f(self, struct, fielddescr): return self.cpu.bh_getfield_raw_f(struct, fielddescr) - opimpl_getfield_raw_i_pure = opimpl_getfield_raw_i - opimpl_getfield_raw_c_pure = opimpl_getfield_raw_c - opimpl_getfield_raw_u_pure = opimpl_getfield_raw_u - opimpl_getfield_raw_r_pure = opimpl_getfield_raw_r - opimpl_getfield_raw_f_pure = opimpl_getfield_raw_f + bhimpl_getfield_raw_i_pure = bhimpl_getfield_raw_i + bhimpl_getfield_raw_c_pure = bhimpl_getfield_raw_c + bhimpl_getfield_raw_u_pure = bhimpl_getfield_raw_u + bhimpl_getfield_raw_r_pure = bhimpl_getfield_raw_r + bhimpl_getfield_raw_f_pure = bhimpl_getfield_raw_f @arguments("self", "r", "d", "i") - def opimpl_setfield_gc_i(self, struct, fielddescr, newvalue): + def bhimpl_setfield_gc_i(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) @arguments("self", "r", "d", "i") - def opimpl_setfield_gc_c(self, struct, fielddescr, newvalue): + def bhimpl_setfield_gc_c(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) @arguments("self", "r", "d", "i") - def opimpl_setfield_gc_u(self, struct, fielddescr, newvalue): + def bhimpl_setfield_gc_u(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) @arguments("self", "r", "d", "r") - def opimpl_setfield_gc_r(self, struct, fielddescr, newvalue): + def bhimpl_setfield_gc_r(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) @arguments("self", "r", "d", "f") - def opimpl_setfield_gc_f(self, struct, fielddescr, newvalue): + def bhimpl_setfield_gc_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) @arguments("self", "i", "d", "i") - def opimpl_setfield_raw_i(self, struct, fielddescr, newvalue): + def bhimpl_setfield_raw_i(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) @arguments("self", "i", "d", "i") - def opimpl_setfield_raw_c(self, struct, fielddescr, newvalue): + def bhimpl_setfield_raw_c(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) @arguments("self", "i", "d", "i") - def opimpl_setfield_raw_u(self, struct, fielddescr, newvalue): + def bhimpl_setfield_raw_u(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) @arguments("self", "i", "d", "r") - def opimpl_setfield_raw_r(self, struct, fielddescr, newvalue): + def bhimpl_setfield_raw_r(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) @arguments("self", "i", "d", "f") - def opimpl_setfield_raw_f(self, struct, fielddescr, newvalue): + def bhimpl_setfield_raw_f(self, struct, fielddescr, newvalue): self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) @arguments("self", "d", returns="r") - def opimpl_new(self, descr): + def bhimpl_new(self, descr): return self.cpu.bh_new(descr) @arguments("self", "d", returns="r") - def opimpl_new_with_vtable(self, descr): + def bhimpl_new_with_vtable(self, descr): return self.cpu.bh_new_with_vtable(descr) @arguments("self", "r", returns="i") - def opimpl_guard_class(self, struct): + def bhimpl_guard_class(self, struct): return self.cpu.bh_classof(struct) @arguments("self", "r", returns="i") - def opimpl_cast_ptr_to_int(self, p): + def bhimpl_cast_ptr_to_int(self, p): return self.cpu.bh_cast_ptr_to_int(p) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 17:02:55 2010 @@ -66,7 +66,7 @@ raise Exception("duplicate entry for op number %d" % value) if key.endswith('_PURE'): key = key[:-5] - name = 'opimpl_' + key.lower() + name = 'bhimpl_' + key.lower() if hasattr(BlackholeInterpreter, name): func = make_execute_function_with_boxes( key.lower(), @@ -78,7 +78,7 @@ cpuclass._execute_by_num_args = execute_by_num_args def make_execute_function_with_boxes(name, func): - # Make a wrapper for 'func'. The func is a simple opimpl_xxx function + # Make a wrapper for 'func'. The func is a simple bhimpl_xxx function # from the BlackholeInterpreter class. The wrapper is a new function # that receives and returns boxed values. for argtype in func.argtypes: From afa at codespeak.net Thu Apr 29 17:06:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 17:06:53 +0200 (CEST) Subject: [pypy-svn] r74229 - in pypy/branch/cpython-extension/pypy/module/cpyext: . include src test Message-ID: <20100429150653.E0C27282BDE@codespeak.net> Author: afa Date: Thu Apr 29 17:06:52 2010 New Revision: 74229 Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/cobject.c (contents, props changed) Removed: pypy/branch/cpython-extension/pypy/module/cpyext/pycobject.py Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py pypy/branch/cpython-extension/pypy/module/cpyext/api.py pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Log: Give up with the rpython-level CPyObject, the memory management was never correct. Since it's used from C only, simply copy the C version. Modified: pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/__init__.py Thu Apr 29 17:06:52 2010 @@ -59,7 +59,6 @@ import pypy.module.cpyext.mapping import pypy.module.cpyext.iterator import pypy.module.cpyext.unicodeobject -import pypy.module.cpyext.pycobject import pypy.module.cpyext.sysmodule import pypy.module.cpyext.number import pypy.module.cpyext.sliceobject Modified: pypy/branch/cpython-extension/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/api.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/api.py Thu Apr 29 17:06:52 2010 @@ -261,6 +261,10 @@ 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', 'init_bufferobject', + 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', + 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', + 'PyCObject_Type', 'init_pycobject', + 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', ] TYPES = {} @@ -476,8 +480,10 @@ def setup_init_functions(eci): init_buffer = rffi.llexternal('init_bufferobject', [], lltype.Void, compilation_info=eci) + init_pycobject = rffi.llexternal('init_pycobject', [], lltype.Void, compilation_info=eci) INIT_FUNCTIONS.extend([ lambda space: init_buffer(), + lambda space: init_pycobject(), ]) def init_function(func): @@ -690,6 +696,7 @@ source_dir / "pythonrun.c", source_dir / "bufferobject.c", source_dir / "object.c", + source_dir / "cobject.c", ], separate_module_sources = [code], export_symbols=export_symbols_eci, Modified: pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/include/pycobject.h Thu Apr 29 17:06:52 2010 @@ -4,6 +4,41 @@ extern "C" { #endif +PyAPI_DATA(PyTypeObject) PyCObject_Type; + +#define PyCObject_Check(op) (op->ob_type == &PyCObject_Type) + +/* Create a PyCObject from a pointer to a C object and an optional + destructor function. If the second argument is non-null, then it + will be called with the first argument if and when the PyCObject is + destroyed. + +*/ +PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtr( + void *cobj, void (*destruct)(void*)); + + +/* Create a PyCObject from a pointer to a C object, a description object, + and an optional destructor function. If the third argument is non-null, + then it will be called with the first and second arguments if and when + the PyCObject is destroyed. +*/ +PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtrAndDesc( + void *cobj, void *desc, void (*destruct)(void*,void*)); + +/* Retrieve a pointer to a C object from a PyCObject. */ +PyAPI_FUNC(void *) PyCObject_AsVoidPtr(PyObject *); + +/* Retrieve a pointer to a description object from a PyCObject. */ +PyAPI_FUNC(void *) PyCObject_GetDesc(PyObject *); + +/* Import a pointer to a C object from a module using a PyCObject. */ +PyAPI_FUNC(void *) PyCObject_Import(char *module_name, char *cobject_name); + +/* Modify a C object. Fails (==0) if object has a destructor. */ +PyAPI_FUNC(int) PyCObject_SetVoidPtr(PyObject *self, void *cobj); + + #if (PY_VERSION_HEX >= 0x02060000 || defined(Py_BUILD_CORE)) typedef struct { PyObject_HEAD Added: pypy/branch/cpython-extension/pypy/module/cpyext/src/cobject.c ============================================================================== --- (empty file) +++ pypy/branch/cpython-extension/pypy/module/cpyext/src/cobject.c Thu Apr 29 17:06:52 2010 @@ -0,0 +1,158 @@ + +/* Wrap void* pointers to be passed between C modules */ + +#include "Python.h" + + +/* Declarations for objects of type PyCObject */ + +typedef void (*destructor1)(void *); +typedef void (*destructor2)(void *, void*); + +PyObject * +PyCObject_FromVoidPtr(void *cobj, void (*destr)(void *)) +{ + PyCObject *self; + + self = PyObject_NEW(PyCObject, &PyCObject_Type); + if (self == NULL) + return NULL; + self->cobject=cobj; + self->destructor=destr; + self->desc=NULL; + + return (PyObject *)self; +} + +PyObject * +PyCObject_FromVoidPtrAndDesc(void *cobj, void *desc, + void (*destr)(void *, void *)) +{ + PyCObject *self; + + if (!desc) { + PyErr_SetString(PyExc_TypeError, + "PyCObject_FromVoidPtrAndDesc called with null" + " description"); + return NULL; + } + self = PyObject_NEW(PyCObject, &PyCObject_Type); + if (self == NULL) + return NULL; + self->cobject = cobj; + self->destructor = (destructor1)destr; + self->desc = desc; + + return (PyObject *)self; +} + +void * +PyCObject_AsVoidPtr(PyObject *self) +{ + if (self) { + if (self->ob_type == &PyCObject_Type) + return ((PyCObject *)self)->cobject; + PyErr_SetString(PyExc_TypeError, + "PyCObject_AsVoidPtr with non-C-object"); + } + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, + "PyCObject_AsVoidPtr called with null pointer"); + return NULL; +} + +void * +PyCObject_GetDesc(PyObject *self) +{ + if (self) { + if (self->ob_type == &PyCObject_Type) + return ((PyCObject *)self)->desc; + PyErr_SetString(PyExc_TypeError, + "PyCObject_GetDesc with non-C-object"); + } + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, + "PyCObject_GetDesc called with null pointer"); + return NULL; +} + +void * +PyCObject_Import(char *module_name, char *name) +{ + PyObject *m, *c; + void *r = NULL; + + if ((m = PyImport_ImportModule(module_name))) { + if ((c = PyObject_GetAttrString(m,name))) { + r = PyCObject_AsVoidPtr(c); + Py_DECREF(c); + } + Py_DECREF(m); + } + return r; +} + +int +PyCObject_SetVoidPtr(PyObject *self, void *cobj) +{ + PyCObject* cself = (PyCObject*)self; + if (cself == NULL || !PyCObject_Check(cself) || + cself->destructor != NULL) { + PyErr_SetString(PyExc_TypeError, + "Invalid call to PyCObject_SetVoidPtr"); + return 0; + } + cself->cobject = cobj; + return 1; +} + +static void +PyCObject_dealloc(PyCObject *self) +{ + if (self->destructor) { + if(self->desc) + ((destructor2)(self->destructor))(self->cobject, self->desc); + else + (self->destructor)(self->cobject); + } + PyObject_DEL(self); +} + + +PyDoc_STRVAR(PyCObject_Type__doc__, +"C objects to be exported from one extension module to another\n\ +\n\ +C objects are used for communication between extension modules. They\n\ +provide a way for an extension module to export a C interface to other\n\ +extension modules, so that extension modules can use the Python import\n\ +mechanism to link to one another."); + +PyTypeObject PyCObject_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "PyCObject", /*tp_name*/ + sizeof(PyCObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)PyCObject_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + 0, /*tp_flags*/ + PyCObject_Type__doc__ /*tp_doc*/ +}; + +void init_pycobject() +{ + PyType_Ready(&PyCObject_Type); +} Modified: pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py ============================================================================== --- pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py (original) +++ pypy/branch/cpython-extension/pypy/module/cpyext/test/test_pycobject.py Thu Apr 29 17:06:52 2010 @@ -1,32 +1,29 @@ -import py +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.pycobject import destructor_short, PyCObject - -class TestPyCObject(BaseApiTest): - def test_pycobject(self, space, api): - ptr = rffi.cast(rffi.VOIDP_real, 1234) - obj = api.PyCObject_FromVoidPtr(ptr, lltype.nullptr(destructor_short.TO)) - assert api.PyCObject_Check(obj) - assert api.PyCObject_AsVoidPtr(obj) == ptr - assert rffi.cast(PyCObject, obj).c_cobject == ptr - api.Py_DecRef(obj) - - obj = api.PyCObject_FromVoidPtrAndDesc(ptr, ptr, - lltype.nullptr(destructor_short.TO)) - api.Py_DecRef(obj) - - def test_pycobject_import(self, space, api): - ptr = rffi.cast(rffi.VOIDP_real, 1234) - obj = api.PyCObject_FromVoidPtr(ptr, lltype.nullptr(destructor_short.TO)) - space.setattr(space.sys, space.wrap("_cpyext_cobject"), obj) - - charp1 = rffi.str2charp("sys") - charp2 = rffi.str2charp("_cpyext_cobject") - assert api.PyCObject_Import(charp1, charp2) == ptr - rffi.free_charp(charp1) - rffi.free_charp(charp2) - - api.Py_DecRef(obj) - space.delattr(space.sys, space.wrap("_cpyext_cobject")) +class AppTestStringObject(AppTestCpythonExtensionBase): + def test_pycobject_import(self): + module = self.import_extension('foo', [ + ("set_ptr", "METH_O", + """ + PyObject *pointer, *module; + void *ptr = PyLong_AsVoidPtr(args); + if (PyErr_Occurred()) return NULL; + pointer = PyCObject_FromVoidPtr(ptr, NULL); + if (PyErr_Occurred()) return NULL; + module = PyImport_ImportModule("foo"); + PyModule_AddObject(module, "_ptr", pointer); + Py_DECREF(module); + if (PyErr_Occurred()) return NULL; + Py_RETURN_NONE; + """), + ("get_ptr", "METH_NOARGS", + """ + void *ptr = PyCObject_Import("foo", "_ptr"); + if (PyErr_Occurred()) return NULL; + return PyLong_FromVoidPtr(ptr); + """)]) + module.set_ptr(1234) + assert "PyCObject object" in str(module._ptr) + import gc; gc.collect() + assert module.get_ptr() == 1234 + del module._ptr From arigo at codespeak.net Thu Apr 29 17:48:49 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 17:48:49 +0200 (CEST) Subject: [pypy-svn] r74230 - in pypy/branch/blackhole-improvement/pypy/jit: backend/llgraph metainterp Message-ID: <20100429154849.16A14282BDE@codespeak.net> Author: arigo Date: Thu Apr 29 17:48:47 2010 New Revision: 74230 Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Port residual_call to pyjitpl. In-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/backend/llgraph/runner.py Thu Apr 29 17:48:47 2010 @@ -29,6 +29,9 @@ self.extrainfo = extrainfo self.name = name + def get_return_type(self): + return self.typeinfo + def get_extra_info(self): return self.extrainfo Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 17:48:47 2010 @@ -9,8 +9,58 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr +from pypy.jit.metainterp.history import INT, REF, FLOAT from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.blackhole import BlackholeInterpreter, NULL + +# ____________________________________________________________ + +def do_call(cpu, argboxes, descr): + # count the number of arguments of the different types + count_i = count_r = count_f = 0 + for i in range(1, len(argboxes)): + type = argboxes[i].type + if type == INT: count_i += 1 + elif type == REF: count_r += 1 + elif type == FLOAT: count_f += 1 + # allocate lists for each type that has at least one argument + if count_i: args_i = [0] * count_i + else: args_i = None + if count_r: args_r = [NULL] * count_r + else: args_r = None + if count_f: args_f = [0.0] * count_f + else: args_f = None + # fill in the lists + count_i = count_r = count_f = 0 + for i in range(1, len(argboxes)): + box = argboxes[i] + if box.type == INT: + args_i[count_i] = box.getint() + count_i += 1 + elif box.type == REF: + args_r[count_r] = box.getptr_base() + count_r += 1 + elif box.type == FLOAT: + args_f[count_f] = box.getfloat() + count_f += 1 + # get the function address as an integer + func = argboxes[0].getint() + # do the call using the correct function from the cpu + rettype = descr.get_return_type() + if rettype == INT: + result = cpu.bh_call_i(func, descr, args_i, args_r, args_f) + return BoxInt(result) + if rettype == REF: + result = cpu.bh_call_r(func, descr, args_i, args_r, args_f) + return BoxPtr(result) + if rettype == FLOAT: + result = cpu.bh_call_f(func, descr, args_i, args_r, args_f) + return BoxFloat(result) + if rettype == 'v': # void + cpu.bh_call_v(func, descr, args_i, args_r, args_f) + return None + raise AssertionError("bad rettype") # ____________________________________________________________ @@ -33,7 +83,6 @@ def make_execute_list(cpuclass): from pypy.jit.backend.model import AbstractCPU - from pypy.jit.metainterp.blackhole import BlackholeInterpreter if 0: # enable this to trace calls to do_xxx def wrap(fn): def myfn(*args): @@ -74,7 +123,11 @@ if func is not None: execute[value] = func continue - pass #XXX... + name = 'do_' + key.lower() + if name in globals(): + execute[value] = globals()[name] + continue + pass # XXX... cpuclass._execute_by_num_args = execute_by_num_args def make_execute_function_with_boxes(name, func): @@ -98,12 +151,14 @@ elif argtype == 'r': value = argbox.getptr_base() elif argtype == 'f': value = argbox.getfloat() newargs = newargs + (value,) + assert not argboxes # result = func(*newargs) # if resulttype == 'i': return BoxInt(result) if resulttype == 'r': return BoxPtr(result) if resulttype == 'f': return BoxFloat(result) + return None # return func_with_new_name(do, 'do_' + name) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/history.py Thu Apr 29 17:48:47 2010 @@ -125,6 +125,12 @@ def _clone_if_mutable(self): return self + def get_return_type(self): + """ Implement in call descr. + Must return INT, REF, FLOAT, or 'v' for void. + """ + raise NotImplementedError + def get_extra_info(self): """ Implement in call descr """ Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 17:48:47 2010 @@ -609,10 +609,29 @@ def opimpl_call(self, callee, varargs): return self.perform_call(callee, varargs) + @arguments("box", "descr", "boxes") + def _opimpl_residual_call1(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) @arguments("box", "descr", "boxes2") - def opimpl_residual_call_ir_i(self, funcbox, calldescr, argboxes): + def _opimpl_residual_call2(self, funcbox, calldescr, argboxes): + return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + @arguments("box", "descr", "boxes3") + def _opimpl_residual_call3(self, funcbox, calldescr, argboxes): return self.do_residual_call(funcbox, calldescr, argboxes, exc=True) + opimpl_residual_call_r_i = _opimpl_residual_call1 + opimpl_residual_call_r_r = _opimpl_residual_call1 + opimpl_residual_call_r_f = _opimpl_residual_call1 + opimpl_residual_call_r_v = _opimpl_residual_call1 + opimpl_residual_call_ir_i = _opimpl_residual_call2 + opimpl_residual_call_ir_r = _opimpl_residual_call2 + opimpl_residual_call_ir_f = _opimpl_residual_call2 + opimpl_residual_call_ir_v = _opimpl_residual_call2 + opimpl_residual_call_irf_i = _opimpl_residual_call3 + opimpl_residual_call_irf_r = _opimpl_residual_call3 + opimpl_residual_call_irf_f = _opimpl_residual_call3 + opimpl_residual_call_irf_v = _opimpl_residual_call3 + @arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) @@ -1016,23 +1035,24 @@ @specialize.arg(1) def execute_varargs(self, opnum, argboxes, descr, exc): - resbox = self.metainterp.execute_and_record_varargs(opnum, argboxes, - descr=descr) - if resbox is not None: - self.make_result_box(resbox) - if exc: - return self.metainterp.handle_exception() - else: - return self.metainterp.assert_no_exception() + return self.metainterp.execute_and_record_varargs(opnum, argboxes, + descr=descr) +## if resbox is not None: +## self.make_result_box(resbox) +## if exc: +## return self.metainterp.handle_exception() +## else: +## return self.metainterp.assert_no_exception() def do_residual_call(self, funcbox, descr, argboxes, exc): + allboxes = [funcbox] + argboxes effectinfo = descr.get_extra_info() if 0:# XXX effectinfo is None or effectinfo.forces_virtual_or_virtualizable: # residual calls require attention to keep virtualizables in-sync self.metainterp.vable_and_vrefs_before_residual_call() # xxx do something about code duplication resbox = self.metainterp.execute_and_record_varargs( - rop.CALL_MAY_FORCE, argboxes, descr=descr) + rop.CALL_MAY_FORCE, allboxes, descr=descr) self.metainterp.vable_and_vrefs_after_residual_call() if resbox is not None: self.make_result_box(resbox) @@ -1042,7 +1062,7 @@ else: return self.metainterp.assert_no_exception() else: - return self.execute_varargs(rop.CALL, argboxes, descr, exc) + return self.execute_varargs(rop.CALL, allboxes, descr, exc) # ____________________________________________________________ From arigo at codespeak.net Thu Apr 29 17:52:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 17:52:59 +0200 (CEST) Subject: [pypy-svn] r74231 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100429155259.F4155282BDE@codespeak.net> Author: arigo Date: Thu Apr 29 17:52:58 2010 New Revision: 74231 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Replace @arguments with @XXX for all functions not ported yet. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 17:52:58 2010 @@ -33,6 +33,11 @@ return func return decorate +def XXX(func): + def cannot_call(*args, **kwds): + raise Exception('not implemented: %s' % func.__name__) + return cannot_call + # ____________________________________________________________ @@ -182,7 +187,7 @@ for _opimpl in ['int_add_ovf', 'int_sub_ovf', 'int_mul_ovf']: exec py.code.Source(''' - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_%s(self, b1, b2): self.execute(rop.%s, b1, b2) return self.metainterp.handle_overflow_error() @@ -200,12 +205,12 @@ ''' % (_opimpl, _opimpl.upper())).compile() @arguments("box") - def opimpl_any_return(self, box): + def _opimpl_any_return(self, box): self.metainterp.finishframe(box) - opimpl_int_return = opimpl_any_return - opimpl_ref_return = opimpl_any_return - opimpl_float_return = opimpl_any_return + opimpl_int_return = _opimpl_any_return + opimpl_ref_return = _opimpl_any_return + opimpl_float_return = _opimpl_any_return @arguments() def opimpl_void_return(self): @@ -215,11 +220,11 @@ def opimpl_catch_exception(self, target): pass # see comment in blackhole.py:opimpl_catch_exception. - @arguments("jumptarget") + @XXX #arguments("jumptarget") def opimpl_goto(self, target): self.pc = target - @arguments("orgpc", "jumptarget", "box", "varargs") + @XXX #arguments("orgpc", "jumptarget", "box", "varargs") def opimpl_goto_if_not(self, pc, target, box, livelist): switchcase = box.getint() if switchcase: @@ -267,7 +272,7 @@ self.load_int() # past the 'box' argument self.ignore_varargs() # past the 'livelist' argument - @arguments("orgpc", "box", "constargs", "jumptargets") + @XXX #arguments("orgpc", "box", "constargs", "jumptargets") def opimpl_switch(self, pc, valuebox, constargs, jumptargets): box = self.implement_guard_value(pc, valuebox) for i in range(len(constargs)): @@ -276,7 +281,7 @@ self.pc = jumptargets[i] break - @arguments("orgpc", "box", "constbox") + @XXX #arguments("orgpc", "box", "constbox") def opimpl_switch_dict(self, pc, valuebox, switchdict): box = self.implement_guard_value(pc, valuebox) search_value = box.getint() @@ -286,57 +291,57 @@ except KeyError: pass - @arguments("descr") + @XXX #arguments("descr") def opimpl_new(self, size): self.execute_with_descr(rop.NEW, descr=size) - @arguments("constbox") + @XXX #arguments("constbox") def opimpl_new_with_vtable(self, vtablebox): self.execute(rop.NEW_WITH_VTABLE, vtablebox) - @arguments("box") + @XXX #arguments("box") def opimpl_runtimenew(self, classbox): self.execute(rop.RUNTIMENEW, classbox) - @arguments("orgpc", "box", "descr") + @XXX #arguments("orgpc", "box", "descr") def opimpl_instanceof(self, pc, objbox, typedescr): clsbox = self.cls_of_box(objbox) if isinstance(objbox, Box): self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) self.execute_with_descr(rop.INSTANCEOF, typedescr, objbox) - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_subclassof(self, box1, box2): self.execute(rop.SUBCLASSOF, box1, box2) - @arguments("descr", "box") + @XXX #arguments("descr", "box") def opimpl_new_array(self, itemsize, countbox): self.execute_with_descr(rop.NEW_ARRAY, itemsize, countbox) - @arguments("box", "descr", "box") + @XXX #arguments("box", "descr", "box") def opimpl_getarrayitem_gc(self, arraybox, arraydesc, indexbox): self.execute_with_descr(rop.GETARRAYITEM_GC, arraydesc, arraybox, indexbox) - @arguments("box", "descr", "box") + @XXX #arguments("box", "descr", "box") def opimpl_getarrayitem_gc_pure(self, arraybox, arraydesc, indexbox): self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydesc, arraybox, indexbox) - @arguments("box", "descr", "box", "box") + @XXX #arguments("box", "descr", "box", "box") def opimpl_setarrayitem_gc(self, arraybox, arraydesc, indexbox, itembox): self.execute_with_descr(rop.SETARRAYITEM_GC, arraydesc, arraybox, indexbox, itembox) - @arguments("box", "descr") + @XXX #arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydesc): self.execute_with_descr(rop.ARRAYLEN_GC, arraydesc, arraybox) - @arguments("descr", "box", "box", "box", "box", "box", "box", "descr") + @XXX #arguments("descr", "box", "box", "box", "box", "box", "box", "descr") def opimpl_arraycopy(self, calldescr, fnptr, sourcebox, destbox, source_startbox, dest_startbox, lengthbox, arraydescr): self.execute_with_descr(rop.ARRAYCOPY, arraydescr, calldescr, fnptr, sourcebox, destbox, source_startbox, dest_startbox, lengthbox) - @arguments("orgpc", "box", "descr", "box") + @XXX #arguments("orgpc", "box", "descr", "box") def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) @@ -349,7 +354,7 @@ rop.INT_ADD, None, indexbox, lenbox) self.make_result_box(indexbox) - @arguments("descr", "descr", "descr", "descr", "box") + @XXX #arguments("descr", "descr", "descr", "descr", "box") def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr, sizebox): sbox = self.metainterp.execute_and_record(rop.NEW, structdescr) @@ -361,20 +366,20 @@ sbox, abox) self.make_result_box(sbox) - @arguments("box", "descr", "descr", "box") + @XXX #arguments("box", "descr", "descr", "box") def opimpl_getlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, itemsdescr, listbox) self.execute_with_descr(rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox) - @arguments("box", "descr", "descr", "box", "box") + @XXX #arguments("box", "descr", "descr", "box", "box") def opimpl_setlistitem_gc(self, listbox, itemsdescr, arraydescr, indexbox, valuebox): arraybox = self.metainterp.execute_and_record(rop.GETFIELD_GC, itemsdescr, listbox) self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox, indexbox, valuebox) - @arguments("orgpc", "box", "descr", "box") + @XXX #arguments("orgpc", "box", "descr", "box") def opimpl_check_resizable_neg_index(self, pc, listbox, lengthdesc, indexbox): negbox = self.metainterp.execute_and_record( @@ -388,7 +393,7 @@ rop.INT_ADD, None, indexbox, lenbox) self.make_result_box(indexbox) - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) @@ -399,7 +404,7 @@ # division by zero! return self.metainterp.raise_zero_division_error() - @arguments("orgpc", "box", "box") + @XXX #arguments("orgpc", "box", "box") def opimpl_check_div_overflow(self, pc, box1, box2): # detect the combination "box1 = -sys.maxint-1, box2 = -1". import sys @@ -416,11 +421,11 @@ # division overflow! return self.metainterp.raise_overflow_error() - @arguments() + @XXX #arguments() def opimpl_overflow_error(self): return self.metainterp.raise_overflow_error() - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) @@ -430,7 +435,7 @@ else: self.execute(rop.INT_NEG, box) - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_oononnull(self, pc, box): value = box.nonnull() if value: @@ -442,7 +447,7 @@ self.generate_guard(pc, opnum, box, []) self.make_result_box(res) - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_ooisnull(self, pc, box): value = box.nonnull() if value: @@ -454,34 +459,34 @@ self.generate_guard(pc, opnum, box, []) self.make_result_box(res) - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_ptr_eq(self, box1, box2): self.execute(rop.OOIS, box1, box2) - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_ptr_ne(self, box1, box2): self.execute(rop.OOISNOT, box1, box2) opimpl_oois = opimpl_ptr_eq opimpl_ooisnot = opimpl_ptr_ne - @arguments("box", "descr") + @XXX #arguments("box", "descr") def opimpl_getfield_gc(self, box, fielddesc): self.execute_with_descr(rop.GETFIELD_GC, fielddesc, box) - @arguments("box", "descr") + @XXX #arguments("box", "descr") def opimpl_getfield_gc_pure(self, box, fielddesc): self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddesc, box) - @arguments("box", "descr", "box") + @XXX #arguments("box", "descr", "box") def opimpl_setfield_gc(self, box, fielddesc, valuebox): self.execute_with_descr(rop.SETFIELD_GC, fielddesc, box, valuebox) - @arguments("box", "descr") + @XXX #arguments("box", "descr") def opimpl_getfield_raw(self, box, fielddesc): self.execute_with_descr(rop.GETFIELD_RAW, fielddesc, box) - @arguments("box", "descr") + @XXX #arguments("box", "descr") def opimpl_getfield_raw_pure(self, box, fielddesc): self.execute_with_descr(rop.GETFIELD_RAW_PURE, fielddesc, box) - @arguments("box", "descr", "box") + @XXX #arguments("box", "descr", "box") def opimpl_setfield_raw(self, box, fielddesc, valuebox): self.execute_with_descr(rop.SETFIELD_RAW, fielddesc, box, valuebox) @@ -511,7 +516,7 @@ vinfo = self.metainterp.staticdata.virtualizable_info return vinfo.array_descrs[index] - @arguments("orgpc", "box", "int") + @XXX #arguments("orgpc", "box", "int") def opimpl_getfield_vable(self, pc, basebox, index): if self._nonstandard_virtualizable(pc, basebox): self.execute_with_descr(rop.GETFIELD_GC, self._get_virtualizable_field_descr(index), basebox) @@ -519,7 +524,7 @@ self.metainterp.check_synchronized_virtualizable() resbox = self.metainterp.virtualizable_boxes[index] self.make_result_box(resbox) - @arguments("orgpc", "box", "int", "box") + @XXX #arguments("orgpc", "box", "int", "box") def opimpl_setfield_vable(self, pc, basebox, index, valuebox): if self._nonstandard_virtualizable(pc, basebox): self.execute_with_descr(rop.SETFIELD_GC, self._get_virtualizable_field_descr(index), basebox, valuebox) @@ -539,7 +544,7 @@ assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) - @arguments("orgpc", "box", "int", "box") + @XXX #arguments("orgpc", "box", "int", "box") def opimpl_getarrayitem_vable(self, pc, basebox, arrayindex, indexbox): if self._nonstandard_virtualizable(pc, basebox): descr = self._get_virtualizable_array_field_descr(arrayindex) @@ -553,7 +558,7 @@ index = self._get_arrayitem_vable_index(pc, arrayindex, indexbox) resbox = self.metainterp.virtualizable_boxes[index] self.make_result_box(resbox) - @arguments("orgpc", "box", "int", "box", "box") + @XXX #arguments("orgpc", "box", "int", "box", "box") def opimpl_setarrayitem_vable(self, pc, basebox, arrayindex, indexbox, valuebox): if self._nonstandard_virtualizable(pc, basebox): @@ -568,7 +573,7 @@ self.metainterp.virtualizable_boxes[index] = valuebox self.metainterp.synchronize_virtualizable() # XXX only the index'th field needs to be synchronized, really - @arguments("orgpc", "box", "int") + @XXX #arguments("orgpc", "box", "int") def opimpl_arraylen_vable(self, pc, basebox, arrayindex): if self._nonstandard_virtualizable(pc, basebox): descr = self._get_virtualizable_array_field_descr(arrayindex) @@ -605,7 +610,7 @@ f.setup_call(varargs) return True - @arguments("bytecode", "varargs") + @XXX #arguments("bytecode", "varargs") def opimpl_call(self, callee, varargs): return self.perform_call(callee, varargs) @@ -632,11 +637,11 @@ opimpl_residual_call_irf_f = _opimpl_residual_call3 opimpl_residual_call_irf_v = _opimpl_residual_call3 - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_call_loopinvariant(self, calldescr, varargs): return self.execute_varargs(rop.CALL_LOOPINVARIANT, varargs, calldescr, exc=True) - @arguments("orgpc", "descr", "varargs") + @XXX #arguments("orgpc", "descr", "varargs") def opimpl_recursive_call(self, pc, calldescr, varargs): warmrunnerstate = self.metainterp.staticdata.state token = None @@ -671,15 +676,15 @@ call_position) return res - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_call_noexception(self, calldescr, varargs): self.do_residual_call(varargs, descr=calldescr, exc=False) - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_call_pure(self, calldescr, varargs): self.execute_varargs(rop.CALL_PURE, varargs, descr=calldescr, exc=False) - @arguments("orgpc", "descr", "box", "varargs") + @XXX #arguments("orgpc", "descr", "box", "varargs") def opimpl_indirect_call(self, pc, calldescr, box, varargs): box = self.implement_guard_value(pc, box) cpu = self.metainterp.cpu @@ -693,7 +698,7 @@ return self.do_residual_call([box] + varargs, descr=calldescr, exc=True) - @arguments("orgpc", "methdescr", "varargs") + @XXX #arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): objbox = varargs[0] clsbox = self.cls_of_box(objbox) @@ -709,56 +714,56 @@ return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=True) - @arguments("box") + @XXX #arguments("box") def opimpl_strlen(self, str): self.execute(rop.STRLEN, str) - @arguments("box") + @XXX #arguments("box") def opimpl_unicodelen(self, str): self.execute(rop.UNICODELEN, str) - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_strgetitem(self, str, index): self.execute(rop.STRGETITEM, str, index) - @arguments("box", "box") + @XXX #arguments("box", "box") def opimpl_unicodegetitem(self, str, index): self.execute(rop.UNICODEGETITEM, str, index) - @arguments("box", "box", "box") + @XXX #arguments("box", "box", "box") def opimpl_strsetitem(self, str, index, newchar): self.execute(rop.STRSETITEM, str, index, newchar) - @arguments("box", "box", "box") + @XXX #arguments("box", "box", "box") def opimpl_unicodesetitem(self, str, index, newchar): self.execute(rop.UNICODESETITEM, str, index, newchar) - @arguments("box") + @XXX #arguments("box") def opimpl_newstr(self, length): self.execute(rop.NEWSTR, length) - @arguments("box") + @XXX #arguments("box") def opimpl_newunicode(self, length): self.execute(rop.NEWUNICODE, length) - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_oosend_canraise(self, methdescr, varargs): return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=True) - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_oosend_noraise(self, methdescr, varargs): self.execute_varargs(rop.OOSEND, varargs, descr=methdescr, exc=False) - @arguments("descr", "varargs") + @XXX #arguments("descr", "varargs") def opimpl_residual_oosend_pure(self, methdescr, boxes): self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_guard_value(self, pc, box): constbox = self.implement_guard_value(pc, box) self.make_result_box(constbox) - @arguments("orgpc", "int") + @XXX #arguments("orgpc", "int") def opimpl_guard_green(self, pc, boxindex): """Like guard_value, but overwrites the original box with the const. Used to prevent Boxes from showing up in the greenkey of some @@ -769,24 +774,24 @@ constbox = self.implement_guard_value(pc, box) self.env[boxindex] = constbox - @arguments("orgpc", "box") + @XXX #arguments("orgpc", "box") def opimpl_guard_class(self, pc, box): clsbox = self.cls_of_box(box) if isinstance(box, Box): self.generate_guard(pc, rop.GUARD_CLASS, box, [clsbox]) self.make_result_box(clsbox) -## @arguments("orgpc", "box", "builtin") +## @XXX #arguments("orgpc", "box", "builtin") ## def opimpl_guard_builtin(self, pc, box, builtin): ## self.generate_guard(pc, "guard_builtin", box, [builtin]) -## @arguments("orgpc", "box", "builtin") +## @XXX #arguments("orgpc", "box", "builtin") ## def opimpl_guard_len(self, pc, box, builtin): ## intbox = self.metainterp.cpu.execute_operation( ## 'len', [builtin.len_func, box], 'int') ## self.generate_guard(pc, "guard_len", box, [intbox]) - @arguments("box") + @XXX #arguments("box") def opimpl_keepalive(self, box): pass # xxx? @@ -816,7 +821,7 @@ else: raise self.metainterp.staticdata.ContinueRunningNormally(varargs) - @arguments() + @XXX #arguments() def opimpl_can_enter_jit(self): # Note: when running with a BlackHole history, this 'can_enter_jit' # may be completely skipped by the logic that replaces perform_call @@ -827,7 +832,7 @@ raise CannotInlineCanEnterJit() self.metainterp.seen_can_enter_jit = True - @arguments() + @XXX #arguments() def opimpl_jit_merge_point(self): if not self.metainterp.is_blackholing(): self.verify_green_args(self.env) @@ -854,15 +859,15 @@ self.metainterp.history.record(rop.DEBUG_MERGE_POINT, [constloc], None) - @arguments("jumptarget") + @XXX #arguments("jumptarget") def opimpl_setup_exception_block(self, exception_target): self.exception_target = exception_target - @arguments() + @XXX #arguments() def opimpl_teardown_exception_block(self): self.exception_target = -1 - @arguments("constbox", "jumptarget", "orgpc") + @XXX #arguments("constbox", "jumptarget", "orgpc") def opimpl_goto_if_exception_mismatch(self, vtableref, next_exc_target, pc): # XXX used to be: # assert isinstance(self.exception_box, Const) # XXX @@ -874,27 +879,27 @@ if not ts.subclassOf(cpu, self.exception_box, vtableref): self.pc = next_exc_target - @arguments("int") + @XXX #arguments("int") def opimpl_put_last_exception(self, index): assert index >= 0 self.env.insert(index, self.exception_box) - @arguments("int") + @XXX #arguments("int") def opimpl_put_last_exc_value(self, index): assert index >= 0 self.env.insert(index, self.exc_value_box) - @arguments() + @XXX #arguments() def opimpl_raise(self): assert len(self.env) == 2 return self.metainterp.finishframe_exception(self.env[0], self.env[1]) - @arguments() + @XXX #arguments() def opimpl_reraise(self): return self.metainterp.finishframe_exception(self.exception_box, self.exc_value_box) - @arguments("box") + @XXX #arguments("box") def opimpl_virtual_ref(self, box): # Details on the content of metainterp.virtualref_boxes: # @@ -928,7 +933,7 @@ metainterp.virtualref_boxes.append(resbox) self.make_result_box(resbox) - @arguments("box") + @XXX #arguments("box") def opimpl_virtual_ref_finish(self, box): # virtual_ref_finish() assumes that we have a stack-like, last-in # first-out order. From arigo at codespeak.net Thu Apr 29 18:31:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Apr 2010 18:31:34 +0200 (CEST) Subject: [pypy-svn] r74232 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100429163134.DBD7B282BE0@codespeak.net> Author: arigo Date: Thu Apr 29 18:31:33 2010 New Revision: 74232 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Minor progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Thu Apr 29 18:31:33 2010 @@ -2,7 +2,6 @@ from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized -from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLException @@ -197,7 +196,7 @@ verbose = self.verbose argtypes = unrolling_iterable(unboundmethod.argtypes) resulttype = unboundmethod.resulttype - handler = func_with_new_name(handler, 'handler_' + name) + handler.func_name = 'handler_' + name return handler def acquire_interp(self): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Thu Apr 29 18:31:33 2010 @@ -6,7 +6,6 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr from pypy.jit.metainterp.history import INT, REF, FLOAT @@ -39,7 +38,7 @@ args_i[count_i] = box.getint() count_i += 1 elif box.type == REF: - args_r[count_r] = box.getptr_base() + args_r[count_r] = box.getref_base() count_r += 1 elif box.type == FLOAT: args_f[count_f] = box.getfloat() @@ -148,7 +147,7 @@ argbox = argboxes[0] argboxes = argboxes[1:] if argtype == 'i': value = argbox.getint() - elif argtype == 'r': value = argbox.getptr_base() + elif argtype == 'r': value = argbox.getref_base() elif argtype == 'f': value = argbox.getfloat() newargs = newargs + (value,) assert not argboxes @@ -160,7 +159,8 @@ if resulttype == 'f': return BoxFloat(result) return None # - return func_with_new_name(do, 'do_' + name) + do.func_name = 'do_' + name + return do def get_execute_funclist(cpu, num_args): # workaround, similar to the next one Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Thu Apr 29 18:31:33 2010 @@ -4,7 +4,6 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.tool.sourcetools import func_with_new_name from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstFloat @@ -238,6 +237,13 @@ # such a way that the 'box' on which we generate the guard is # typically not included in the livelist. + @arguments("label", "box", "box") + def opimpl_goto_if_not_int_lt(self, target, box1, box2): + if box1.getint() < box2.getint(): + pass + else: + self.pc = target + def follow_jump(self): _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not assert ord(self.bytecode[self.pc]) == _op_goto_if_not @@ -758,10 +764,9 @@ def opimpl_residual_oosend_pure(self, methdescr, boxes): self.execute_varargs(rop.OOSEND_PURE, boxes, descr=methdescr, exc=False) - @XXX #arguments("orgpc", "box") - def opimpl_guard_value(self, pc, box): - constbox = self.implement_guard_value(pc, box) - self.make_result_box(constbox) + @arguments("orgpc", "box") + def opimpl_int_guard_value(self, pc, box): + return self.implement_guard_value(pc, box) @XXX #arguments("orgpc", "int") def opimpl_guard_green(self, pc, boxindex): @@ -1259,7 +1264,7 @@ self.free_frames_list = [] def is_blackholing(self): - return self.history is None + return False # XXX get rid of this method def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: @@ -2067,6 +2072,7 @@ args = () next_argcode = 0 code = self.bytecode + orgpc = position position += 1 for argtype in argtypes: if argtype == "box": # a box, of whatever type @@ -2127,6 +2133,8 @@ argcodes[next_argcode + 2]) next_argcode = next_argcode + 3 position = position3 + 1 + length3 + elif argtype == "orgpc": + value = orgpc else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) @@ -2157,5 +2165,5 @@ # unboundmethod = getattr(MIFrame, 'opimpl_' + name).im_func argtypes = unrolling_iterable(unboundmethod.argtypes) - handler = func_with_new_name(handler, 'handler_' + name) + handler.func_name = 'handler_' + name return handler From fijal at codespeak.net Thu Apr 29 20:15:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:15:20 +0200 (CEST) Subject: [pypy-svn] r74235 - pypy/branch/chunked-list Message-ID: <20100429181520.CB93B282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:15:19 2010 New Revision: 74235 Added: pypy/branch/chunked-list/ (props changed) - copied from r74234, pypy/trunk/ Log: An attempt to solve gc problems with large lists of gc pointers (again) From fijal at codespeak.net Thu Apr 29 20:37:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:37:20 +0200 (CEST) Subject: [pypy-svn] r74236 - in pypy/trunk/pypy: . _interfaces annotation annotation/test config doc/config interpreter interpreter/test lib module/cpyext module/cpyext/include module/cpyext/src module/cpyext/test module/imp objspace/std objspace/std/test rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test tool translator translator/c translator/c/gcc translator/c/gcc/test translator/c/src translator/c/test translator/goal translator/platform translator/test translator/tool translator/tool/test Message-ID: <20100429183720.9C65C282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:37:15 2010 New Revision: 74236 Added: pypy/trunk/pypy/_interfaces/ (props changed) - copied from r74235, pypy/branch/cpython-extension/pypy/_interfaces/ pypy/trunk/pypy/doc/config/objspace.usemodules.cpyext.txt - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/doc/config/objspace.usemodules.cpyext.txt pypy/trunk/pypy/doc/config/translation.shared.txt - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/doc/config/translation.shared.txt pypy/trunk/pypy/module/cpyext/ (props changed) - copied from r74235, pypy/branch/cpython-extension/pypy/module/cpyext/ pypy/trunk/pypy/rlib/_rweakkeydict.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/_rweakkeydict.py pypy/trunk/pypy/rlib/_rweakvaldict.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/_rweakvaldict.py pypy/trunk/pypy/rlib/entrypoint.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/entrypoint.py pypy/trunk/pypy/rlib/exports.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/exports.py pypy/trunk/pypy/rlib/test/test_rweakkeydict.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/test/test_rweakkeydict.py pypy/trunk/pypy/rlib/test/test_rweakvaldict.py - copied unchanged from r74235, pypy/branch/cpython-extension/pypy/rlib/test/test_rweakvaldict.py Removed: pypy/trunk/pypy/rlib/rweakrefimpl.py pypy/trunk/pypy/rlib/test/test_rweakref.py Modified: pypy/trunk/pypy/annotation/annrpython.py pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/test/test_annrpython.py pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/conftest.py pypy/trunk/pypy/interpreter/argument.py pypy/trunk/pypy/interpreter/test/test_argument.py pypy/trunk/pypy/interpreter/test/test_typedef.py pypy/trunk/pypy/interpreter/typedef.py pypy/trunk/pypy/lib/identity_dict.py pypy/trunk/pypy/module/cpyext/include/ (props changed) pypy/trunk/pypy/module/cpyext/src/ (props changed) pypy/trunk/pypy/module/cpyext/test/ (props changed) pypy/trunk/pypy/module/imp/importing.py pypy/trunk/pypy/objspace/std/fake.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/test/test_setobject.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/rlib/rstring.py pypy/trunk/pypy/rlib/rweakref.py pypy/trunk/pypy/rlib/test/test_rarithmetic.py pypy/trunk/pypy/rlib/test/test_rstring.py pypy/trunk/pypy/rpython/annlowlevel.py pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/lltype.py pypy/trunk/pypy/rpython/lltypesystem/rdict.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/trunk/pypy/tool/fixeol pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/trunk/pypy/translator/c/gcc/trackgcroot.py pypy/trunk/pypy/translator/c/genc.py pypy/trunk/pypy/translator/c/node.py pypy/trunk/pypy/translator/c/src/commondefs.h pypy/trunk/pypy/translator/c/src/main.h pypy/trunk/pypy/translator/c/src/obmalloc.c pypy/trunk/pypy/translator/c/test/test_genc.py pypy/trunk/pypy/translator/c/test/test_newgc.py pypy/trunk/pypy/translator/c/test/test_standalone.py pypy/trunk/pypy/translator/driver.py pypy/trunk/pypy/translator/goal/ann_override.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/linux.py pypy/trunk/pypy/translator/platform/posix.py pypy/trunk/pypy/translator/platform/windows.py pypy/trunk/pypy/translator/test/test_unsimplify.py pypy/trunk/pypy/translator/tool/cbuild.py pypy/trunk/pypy/translator/tool/test/test_cbuild.py pypy/trunk/pypy/translator/unsimplify.py Log: (xoraxax, afa, agaynor, arigo, benjamin, exarkun, fijal, jandem, lucian, trundle, zooko) Merge the cpyext branch - this branch adds an ability to load cpython extension modules written in C (.so). Work in progress, has known bugs. Right now the module is disabled since it adds an attribute on W_Root (_pyolifeline) which is not a desired behavior. Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Thu Apr 29 20:37:15 2010 @@ -87,12 +87,12 @@ """Recursively build annotations about the specific entry point.""" assert isinstance(function, types.FunctionType), "fix that!" + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() # make input arguments and set their type - inputcells = [self.typeannotation(t) for t in input_arg_types] + args_s = [self.typeannotation(t) for t in input_arg_types] - desc = self.bookkeeper.getdesc(function) - desc.getcallfamily() # record this implicit call (hint for back-ends) - flowgraph = desc.specialize(inputcells) + flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) if not isinstance(flowgraph, FunctionGraph): assert isinstance(flowgraph, annmodel.SomeObject) return flowgraph Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Thu Apr 29 20:37:15 2010 @@ -209,8 +209,9 @@ if len(self._cache) != 1: raise NoStandardGraph(self) [graph] = self._cache.values() + relax_sig_check = getattr(self.pyobj, "relax_sig_check", False) if (graph.signature != self.signature or - graph.defaults != self.defaults): + graph.defaults != self.defaults) and not relax_sig_check: raise NoStandardGraph(self) return graph Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Thu Apr 29 20:37:15 2010 @@ -3185,6 +3185,13 @@ assert isinstance(s, annmodel.SomeList) assert s.listdef.listitem.resized + def test_varargs(self): + def f(*args): + return args[0] + 42 + a = self.RPythonAnnotator() + s = a.build_types(f, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + def test_listitem_no_mutating(self): from pypy.rlib.debug import check_annotation called = [] @@ -3304,6 +3311,15 @@ s = a.build_types(f, [int]) assert s.knowntype is int + def test_relax(self): + def f(*args): + return args[0] + args[1] + f.relax_sig_check = True + def g(x): + return f(x, x - x) + a = self.RPythonAnnotator() + s = a.build_types(g, [int]) + assert a.bookkeeper.getdesc(f).getuniquegraph() def g(n): return [0,1,2,n] Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Thu Apr 29 20:37:15 2010 @@ -29,7 +29,8 @@ "rctime" , "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", - "thread", "itertools", "pyexpat", "_ssl"] + "thread", "itertools", "pyexpat", "_ssl"] # "cpyext"] commented out until + # it stops adding _pyolifeline on W_Root )) working_oo_modules = default_modules.copy() @@ -62,11 +63,14 @@ module_dependencies = {} -module_suggests = { # the reason you want _rawffi is for ctypes, which - # itself needs the interp-level struct module - # because 'P' is missing from the app-level one - '_rawffi': [("objspace.usemodules.struct", True)], - } +module_suggests = { + # the reason you want _rawffi is for ctypes, which + # itself needs the interp-level struct module + # because 'P' is missing from the app-level one + "_rawffi": [("objspace.usemodules.struct", True)], + "cpyext": [("translation.secondaryentrypoints", "cpyext"), + ("translation.shared", sys.platform == "win32")], + } module_import_dependencies = { # no _rawffi if importing pypy.rlib.libffi raises ImportError Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Thu Apr 29 20:37:15 2010 @@ -42,6 +42,9 @@ }, cmdline="-b --backend"), + BoolOption("shared", "Build as a shared library", + default=False, cmdline="--shared"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", default=True, cmdline="--log"), @@ -138,6 +141,9 @@ ArbitraryOption("instrumentctl", "internal", default=None), StrOption("output", "Output file name", cmdline="--output"), + StrOption("secondaryentrypoints", + "Comma separated list of keys choosing secondary entrypoints", + cmdline="--entrypoints", default=""), BoolOption("dump_static_data_info", "Dump static data info", cmdline="--dump_static_data_info", Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Thu Apr 29 20:37:15 2010 @@ -59,7 +59,7 @@ try: return _SPACECACHE[key] except KeyError: - if option.runappdirect: + if getattr(option, 'runappdirect', None): if name not in (None, 'std'): myname = getattr(sys, 'pypy_objspaceclass', '') if not myname.lower().startswith(name): Modified: pypy/trunk/pypy/interpreter/argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/argument.py (original) +++ pypy/trunk/pypy/interpreter/argument.py Thu Apr 29 20:37:15 2010 @@ -128,19 +128,6 @@ kwds_w[self.keywords[i]] = self.keywords_w[i] return self.arguments_w, kwds_w - def unpack_cpy(self, starting_at=0): - assert starting_at >= 0 - space = self.space - args_w = self.arguments_w - w_kw = space.newdict() - if self.keywords: - for i in range(len(self.keywords)): - space.setitem(w_kw, space.wrap(self.keywords[i]), self.keywords_w[i]) - if starting_at != 0: - args_w = args_w[starting_at:] - args_tuple = space.newtuple([space.wrap(args_w), w_kw]) - return args_tuple - def replace_arguments(self, args_w): "Return a new Arguments with a args_w as positional arguments." return Arguments(self.space, args_w, self.keywords, self.keywords_w) Modified: pypy/trunk/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_argument.py (original) +++ pypy/trunk/pypy/interpreter/test/test_argument.py Thu Apr 29 20:37:15 2010 @@ -135,12 +135,6 @@ assert args1.keywords is args.keywords assert args1.keywords_w is args.keywords_w - def test_unpack_cpy(self): - space = DummySpace() - args = Arguments(space, ["0"]) - assert space.eq_w(args.unpack_cpy(), space.newtuple([space.newlist([space.wrap("0")]), space.newdict()])) - assert space.eq_w(args.unpack_cpy(1), space.newtuple([space.newlist(), space.newdict()])) - def test_fixedunpacked(self): space = DummySpace() Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_typedef.py (original) +++ pypy/trunk/pypy/interpreter/test/test_typedef.py Thu Apr 29 20:37:15 2010 @@ -1,5 +1,6 @@ from pypy.interpreter import typedef from pypy.tool.udir import udir +from pypy.interpreter.baseobjspace import Wrappable # this test isn't so much to test that the objspace interface *works* # -- it's more to test that it's *there* @@ -132,6 +133,17 @@ assert len(set) <= 6, "%s has %d subclasses:\n%r" % ( cls, len(set), [subcls.__name__ for subcls in set]) + def test_getsetproperty(self): + class W_SomeType(Wrappable): + pass + def fget(self, space, w_self): + assert self is prop + W_SomeType.typedef = typedef.TypeDef( + 'some_type', + x=typedef.GetSetProperty(fget, use_closure=True)) + w_obj = self.space.wrap(W_SomeType()) + assert self.space.getattr(w_obj, self.space.wrap('x')) is None + class AppTestTypeDef: Modified: pypy/trunk/pypy/interpreter/typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/typedef.py (original) +++ pypy/trunk/pypy/interpreter/typedef.py Thu Apr 29 20:37:15 2010 @@ -9,7 +9,7 @@ DescrMismatch from pypy.interpreter.error import OperationError, operationerrfmt from pypy.tool.sourcetools import compile2, func_with_new_name -from pypy.rlib.objectmodel import instantiate, compute_identity_hash +from pypy.rlib.objectmodel import instantiate, compute_identity_hash, specialize from pypy.rlib.jit import hint class TypeDef: @@ -301,44 +301,63 @@ # ____________________________________________________________ -def make_descr_typecheck_wrapper(func, extraargs=(), cls=None): + at specialize.arg(0) +def make_descr_typecheck_wrapper(tag, func, extraargs=(), cls=None, + use_closure=False): if func is None: return None - if cls is None: + return _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure) + + at specialize.memo() +def _make_descr_typecheck_wrapper(tag, func, extraargs, cls, use_closure): + # - if cls is None, the wrapped object is passed to the function + # - if cls is a class, an unwrapped instance is passed + # - if cls is a string, XXX unused? + if cls is None and use_closure: return func if hasattr(func, 'im_func'): assert func.im_class is cls func = func.im_func - + miniglobals = { func.__name__: func, 'OperationError': OperationError } if isinstance(cls, str): + assert 0, "unused?" #print " value Read the value of the property of the given obj.""" @@ -400,7 +424,7 @@ return space.wrap(property) else: try: - return property.fget(space, w_obj) + return property.fget(property, space, w_obj) except DescrMismatch, e: return w_obj.descr_call_mismatch(space, '__getattribute__',\ property.reqcls, Arguments(space, [w_obj, @@ -414,7 +438,7 @@ raise OperationError(space.w_TypeError, space.wrap("readonly attribute")) try: - fset(space, w_obj, w_value) + fset(property, space, w_obj, w_value) except DescrMismatch, e: w_obj.descr_call_mismatch(space, '__setattr__',\ property.reqcls, Arguments(space, [w_obj, @@ -428,7 +452,7 @@ raise OperationError(space.w_AttributeError, space.wrap("cannot delete attribute")) try: - fdel(space, w_obj) + fdel(property, space, w_obj) except DescrMismatch, e: w_obj.descr_call_mismatch(space, '__delattr__',\ property.reqcls, Arguments(space, [w_obj, Modified: pypy/trunk/pypy/lib/identity_dict.py ============================================================================== --- pypy/trunk/pypy/lib/identity_dict.py (original) +++ pypy/trunk/pypy/lib/identity_dict.py Thu Apr 29 20:37:15 2010 @@ -30,6 +30,12 @@ def __contains__(self, arg): return id(arg) in self._dict + def copy(self): + d = type(self)() + d.update(self.iteritems()) + assert len(d) == len(self) + return d + class IdentityDictPyPy(object, DictMixin): __slots__ = ["_dict"] @@ -52,6 +58,11 @@ def __contains__(self, arg): return arg in self._dict + def copy(self): + d = type(self)() + d.update(self.iteritems()) + assert len(d) == len(self) + return d if idict is None: identity_dict = IdentityDictPurePython Modified: pypy/trunk/pypy/module/imp/importing.py ============================================================================== --- pypy/trunk/pypy/module/imp/importing.py (original) +++ pypy/trunk/pypy/module/imp/importing.py Thu Apr 29 20:37:15 2010 @@ -25,6 +25,11 @@ # PY_CODERESOURCE = 8 IMP_HOOK = 9 +if sys.platform.startswith('win'): + so_extension = ".pyd" +else: + so_extension = ".so" + def find_modtype(space, filepart): """Check which kind of module to import for the given filepart, which is a path without extension. Returns PY_SOURCE, PY_COMPILED or @@ -40,16 +45,18 @@ # look for a lone .pyc file. # The "imp" module does not respect this, and is allowed to find # lone .pyc files. - if not space.config.objspace.lonepycfiles: - return SEARCH_ERROR, None, None - # check the .pyc file - if space.config.objspace.usepycfiles: + if space.config.objspace.usepycfiles and space.config.objspace.lonepycfiles: pycfile = filepart + ".pyc" if os.path.exists(pycfile) and case_ok(pycfile): # existing .pyc file return PY_COMPILED, ".pyc", "rb" + if space.config.objspace.usemodules.cpyext: + pydfile = filepart + so_extension + if os.path.exists(pydfile) and case_ok(pydfile): + return C_EXTENSION, so_extension, "rb" + return SEARCH_ERROR, None, None if sys.platform in ['linux2', 'freebsd']: @@ -332,6 +339,9 @@ except: stream.close() raise + if modtype == C_EXTENSION: + filename = filepart + suffix + return FindInfo(modtype, filename, None, suffix, filemode) except StreamErrors: pass @@ -356,7 +366,7 @@ if find_info.modtype == C_BUILTIN: return space.getbuiltinmodule(find_info.filename, force_init=True) - if find_info.modtype in (PY_SOURCE, PY_COMPILED, PKG_DIRECTORY): + if find_info.modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION, PKG_DIRECTORY): w_mod = None if reuse: try: @@ -397,6 +407,12 @@ # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod + elif find_info.modtype == C_EXTENSION and space.config.objspace.usemodules.cpyext: + # the next line is mandantory to init cpyext + space.getbuiltinmodule("cpyext") + from pypy.module.cpyext.api import load_extension_module + load_extension_module(space, find_info.filename, space.str_w(w_modulename)) + return check_sys_modules(space, w_modulename) except OperationError: w_mods = space.sys.get('modules') space.call_method(w_mods, 'pop', w_modulename, space.w_None) Modified: pypy/trunk/pypy/objspace/std/fake.py ============================================================================== --- pypy/trunk/pypy/objspace/std/fake.py (original) +++ pypy/trunk/pypy/objspace/std/fake.py Thu Apr 29 20:37:15 2010 @@ -114,6 +114,9 @@ cpy_type.__name__, base, **kw) def __init__(w_self, space, val): w_self.val = val + w_self.space = space + def getdict(w_self): + return w_self.space.wrap(w_self.val.__dict__) def unwrap(w_self, space): return w_self.val W_Fake.__name__ = 'W_Fake%s'%(cpy_type.__name__.capitalize()) Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Thu Apr 29 20:37:15 2010 @@ -7,7 +7,7 @@ from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model, transparent, callmethod, proxyobject) from pypy.objspace.descroperation import DescrOperation, raiseattrerror -from pypy.rlib.objectmodel import instantiate +from pypy.rlib.objectmodel import instantiate, r_dict from pypy.rlib.debug import make_sure_not_resized from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import we_are_translated @@ -206,8 +206,11 @@ return W_ComplexObject(x.real, x.imag) if isinstance(x, set): - wrappeditems = [self.wrap(item) for item in x] - return W_SetObject(self, wrappeditems) + rdict_w = r_dict(self.eq_w, self.hash_w) + for item in x: + rdict_w[self.wrap(item)] = None + res = W_SetObject(self, rdict_w) + return res if isinstance(x, frozenset): wrappeditems = [self.wrap(item) for item in x] Modified: pypy/trunk/pypy/objspace/std/test/test_setobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_setobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_setobject.py Thu Apr 29 20:37:15 2010 @@ -46,6 +46,8 @@ t = W_SetObject(self.space, None) _initialize_set(self.space, t, self.word) assert self.space.eq_w(s,t) + u = self.space.wrap(set('simsalabim')) + assert self.space.eq_w(s,u) class AppTestAppSetTest: def test_subtype(self): Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Thu Apr 29 20:37:15 2010 @@ -14,6 +14,7 @@ from pypy.rlib.rarithmetic import intmask, r_uint from copy_reg import _HEAPTYPE +_CPYTYPE = 1 # used for non-heap types defined in C # from compiler/misc.py @@ -96,7 +97,7 @@ w_self.needsdel = False w_self.weakrefable = False w_self.weak_subclasses = [] - w_self.__flags__ = 0 # or _HEAPTYPE + w_self.__flags__ = 0 # or _HEAPTYPE or _CPYTYPE w_self.instancetypedef = overridetypedef if overridetypedef is not None: @@ -353,6 +354,9 @@ def is_heaptype(w_self): return w_self.__flags__&_HEAPTYPE + def is_cpytype(w_self): + return w_self.__flags__ & _CPYTYPE + def get_module(w_self): space = w_self.space if w_self.is_heaptype() and '__module__' in w_self.dict_w: Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Thu Apr 29 20:37:15 2010 @@ -126,6 +126,8 @@ raise OverflowError def compute_restype(self_type, other_type): + if self_type is other_type: + return self_type if other_type in (bool, int, long): if self_type is bool: return int Modified: pypy/trunk/pypy/rlib/rstring.py ============================================================================== --- pypy/trunk/pypy/rlib/rstring.py (original) +++ pypy/trunk/pypy/rlib/rstring.py Thu Apr 29 20:37:15 2010 @@ -1,11 +1,49 @@ - -""" String builder interface +""" String builder interface and string functions """ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation.model import SomeObject, SomeString, s_None,\ SomeChar, SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString + +# -------------- public API for string functions ----------------------- +def split(value, by, maxsplit=-1): + bylen = len(by) + if bylen == 0: + raise ValueError("empty separator") + + res = [] + start = 0 + while maxsplit != 0: + next = value.find(by, start) + if next < 0: + break + res.append(value[start:next]) + start = next + bylen + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + res.append(value[start:len(value)]) + return res + +def rsplit(value, by, maxsplit=-1): + res = [] + end = len(value) + bylen = len(by) + if bylen == 0: + raise ValueError("empty separator") + + while maxsplit != 0: + next = value.rfind(by, 0, end) + if next < 0: + break + res.append(value[next+bylen:end]) + end = next + maxsplit -= 1 # NB. if it's already < 0, it stays < 0 + + res.append(value[:end]) + res.reverse() + return res + # -------------- public API --------------------------------- INIT_SIZE = 100 # XXX tweak Modified: pypy/trunk/pypy/rlib/rweakref.py ============================================================================== --- pypy/trunk/pypy/rlib/rweakref.py (original) +++ pypy/trunk/pypy/rlib/rweakref.py Thu Apr 29 20:37:15 2010 @@ -1,6 +1,7 @@ """ Weakref support in RPython. Supports ref() without callbacks, -and a limited version of WeakValueDictionary. LLType only for now! +a form of WeakKeyDictionary, and a limited version of WeakValueDictionary. +LLType only for now! """ import weakref @@ -27,6 +28,36 @@ self._dict[key] = value +class RWeakKeyDictionary(object): + """A dictionary containing weak keys. + Keys and values must be instances. + Prebuilt RWeakKeyDictionaries must be empty. + """ + + def __init__(self, keyclass, valueclass): + self._dict = weakref.WeakKeyDictionary() + self._keyclass = keyclass + self._valueclass = valueclass + + def get(self, key): + """Get the value associated to 'key', or None by default.""" + assert isinstance(key, self._keyclass) + return self._dict.get(key, None) + + def set(self, key, value): + """Set the key/value pair (or delete it if value is None).""" + assert isinstance(key, self._keyclass) + if value is None: + self._dict.pop(key, None) + else: + assert isinstance(value, self._valueclass) + self._dict[key] = value + + def length(self): + """Mostly for debugging. Slow, don't use in real code.""" + return len(self._dict) + + # ____________________________________________________________ from pypy.rpython import extregistry @@ -41,8 +72,8 @@ self.valueclassdef = valueclassdef def rtyper_makerepr(self, rtyper): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.WeakValueDictRepr(rtyper) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.WeakValueDictRepr(rtyper) def rtyper_makekey_ex(self, rtyper): return self.__class__, @@ -65,14 +96,11 @@ _about_ = RWeakValueDictionary def compute_result_annotation(self, s_valueclass): - assert isinstance(s_valueclass, annmodel.SomePBC) - assert s_valueclass.is_constant() - [desc] = s_valueclass.descriptions - return SomeWeakValueDict(desc.getuniqueclassdef()) + return SomeWeakValueDict(_getclassdef(s_valueclass)) def specialize_call(self, hop): - from pypy.rlib import rweakrefimpl - return rweakrefimpl.specialize_make_weakdict(hop) + from pypy.rlib import _rweakvaldict + return _rweakvaldict.specialize_make_weakdict(hop) class Entry(extregistry.ExtRegistryEntry): _type_ = RWeakValueDictionary @@ -81,3 +109,65 @@ bk = self.bookkeeper x = self.instance return SomeWeakValueDict(bk.getuniqueclassdef(x._valueclass)) + +def _getclassdef(s_instance): + assert isinstance(s_instance, annmodel.SomePBC) + assert s_instance.is_constant() + [desc] = s_instance.descriptions + return desc.getuniqueclassdef() + +# ____________________________________________________________ + +class SomeWeakKeyDict(annmodel.SomeObject): + knowntype = RWeakKeyDictionary + + def __init__(self, keyclassdef, valueclassdef): + self.keyclassdef = keyclassdef + self.valueclassdef = valueclassdef + + def rtyper_makerepr(self, rtyper): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.WeakKeyDictRepr(rtyper) + + def rtyper_makekey_ex(self, rtyper): + return self.__class__, + + def method_get(self, s_key): + assert isinstance(s_key, annmodel.SomeInstance) + assert s_key.classdef.issubclass(self.keyclassdef) + return annmodel.SomeInstance(self.valueclassdef, can_be_None=True) + + def method_set(self, s_key, s_value): + s_oldvalue = self.method_get(s_key) + assert s_oldvalue.contains(s_value) + + def method_length(self): + return annmodel.SomeInteger(nonneg=True) + +class __extend__(pairtype(SomeWeakKeyDict, SomeWeakKeyDict)): + def union((s_wkd1, s_wkd2)): + if s_wkd1.keyclassdef is not s_wkd2.keyclassdef: + return SomeObject() # not the same key class! complain... + if s_wkd1.valueclassdef is not s_wkd2.valueclassdef: + return SomeObject() # not the same value class! complain... + return SomeWeakKeyDict(s_wkd1.keyclassdef, s_wkd1.valueclassdef) + +class Entry(extregistry.ExtRegistryEntry): + _about_ = RWeakKeyDictionary + + def compute_result_annotation(self, s_keyclass, s_valueclass): + return SomeWeakKeyDict(_getclassdef(s_keyclass), + _getclassdef(s_valueclass)) + + def specialize_call(self, hop): + from pypy.rlib import _rweakkeydict + return _rweakkeydict.specialize_make_weakdict(hop) + +class Entry(extregistry.ExtRegistryEntry): + _type_ = RWeakKeyDictionary + + def compute_annotation(self): + bk = self.bookkeeper + x = self.instance + return SomeWeakKeyDict(bk.getuniqueclassdef(x._keyclass), + bk.getuniqueclassdef(x._valueclass)) Modified: pypy/trunk/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/test/test_rarithmetic.py Thu Apr 29 20:37:15 2010 @@ -365,3 +365,7 @@ def test_isnan(): assert isnan(NAN) + +def test_int_real_union(): + from pypy.rpython.lltypesystem.rffi import r_int_real + assert compute_restype(r_int_real, r_int_real) is r_int_real Modified: pypy/trunk/pypy/rlib/test/test_rstring.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rstring.py (original) +++ pypy/trunk/pypy/rlib/test/test_rstring.py Thu Apr 29 20:37:15 2010 @@ -1,5 +1,26 @@ -from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit + +def test_split(): + assert split("", 'x') == [''] + assert split("a", "a", 1) == ['', ''] + assert split(" ", " ", 1) == ['', ''] + assert split("aa", "a", 2) == ['', '', ''] + assert split('a|b|c|d', '|') == ['a', 'b', 'c', 'd'] + assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d'] + assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] + assert split('endcase test', 'test') == ['endcase ', ''] + raises(ValueError, split, 'abc', '') + +def test_rsplit(): + assert rsplit("a", "a", 1) == ['', ''] + assert rsplit(" ", " ", 1) == ['', ''] + assert rsplit("aa", "a", 2) == ['', '', ''] + assert rsplit('a|b|c|d', '|') == ['a', 'b', 'c', 'd'] + assert rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd'] + assert rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] + assert rsplit('endcase test', 'test') == ['endcase ', ''] + raises(ValueError, rsplit, "abc", '') def test_string_builder(): s = StringBuilder() Modified: pypy/trunk/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/trunk/pypy/rpython/annlowlevel.py (original) +++ pypy/trunk/pypy/rpython/annlowlevel.py Thu Apr 29 20:37:15 2010 @@ -34,6 +34,7 @@ else: s = compact() return s + 'Const' + __repr__ = __str__ class LowLevelAnnotatorPolicy(AnnotatorPolicy): allow_someobjects = False Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Thu Apr 29 20:37:15 2010 @@ -675,7 +675,8 @@ if T is lltype.Void: return None if isinstance(T, lltype.Ptr): - if not cobj: # NULL pointer + if not cobj or not ctypes.cast(cobj, ctypes.c_void_p).value: # NULL pointer + # CFunctionType.__nonzero__ is broken before Python 2.6 return lltype.nullptr(T.TO) if isinstance(T.TO, lltype.Struct): REAL_TYPE = T.TO @@ -963,7 +964,7 @@ def invoke_via_ctypes(*argvalues): global _callback_exc_info cargs = [] - for i in range(len(FUNCTYPE.ARGS)): + for i in range(len(argvalues)): if i not in void_arguments: cvalue = lltype2ctypes(argvalues[i]) if i in container_arguments: Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Thu Apr 29 20:37:15 2010 @@ -1,3 +1,8 @@ +import StringIO +import traceback +import sys + +import py from pypy.rlib.rarithmetic import (r_int, r_uint, intmask, r_singlefloat, r_ulonglong, r_longlong, base_int, normalizedinttype) @@ -11,6 +16,7 @@ import weakref TLS = tlsobject() +TRACK_ALLOCATIONS = False class _uninitialized(object): def __init__(self, TYPE): @@ -1312,27 +1318,49 @@ def _was_freed(self): return False +ALLOCATED = identity_dict() + class _parentable(_container): _kind = "?" __slots__ = ('_TYPE', '_parent_type', '_parent_index', '_keepparent', '_wrparent', - '__weakref__', - '_storage') + '__weakref__', '_traceback', + '__storage') - def __init__(self, TYPE): + def __init__(self, TYPE, track_allocation=None): self._wrparent = None self._TYPE = TYPE self._storage = True # means "use default storage", as opposed to: # None - container was freed # - using ctypes # (see ll2ctypes.py) + if track_allocation is not False and TRACK_ALLOCATIONS: + self._traceback = self._get_traceback() + ALLOCATED[self] = None + else: + self._traceback = None + + def _get_traceback(self): + frame = sys._getframe().f_back.f_back.f_back.f_back + sio = StringIO.StringIO() + traceback.print_stack(frame, file=sio) + return sio.getvalue() def _free(self): self._check() # no double-frees self._storage = None + def _storage_get(self): + return self.__storage + + def _storage_set(self, value): + self.__storage = value + if value is not True and self in ALLOCATED: + del ALLOCATED[self] + _storage = property(_storage_get, _storage_set) + def _was_freed(self): if self._storage is None: return True @@ -1409,14 +1437,14 @@ class _struct(_parentable): _kind = "structure" - __slots__ = ('_hash_cache_',) + __slots__ = ('_hash_cache_', '_compilation_info') - def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): + def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) - def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): - _parentable.__init__(self, TYPE) + def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None, track_allocation=None): + _parentable.__init__(self, TYPE, track_allocation) if n is not None and TYPE._arrayfld is None: raise TypeError("%r is not variable-sized" % (TYPE,)) if n is None and TYPE._arrayfld is not None: @@ -1485,12 +1513,12 @@ __slots__ = ('items',) - def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None): + def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None, track_allocation=None): if not isinstance(n, int): raise TypeError, "array length must be an int" if n < 0: raise ValueError, "negative array length" - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation) try: myrange = range(n) except OverflowError: @@ -1557,7 +1585,7 @@ _cache = weakref.WeakKeyDictionary() # parentarray -> {subarrays} def __init__(self, TYPE, parent, baseoffset_or_fieldname): - _parentable.__init__(self, TYPE) + _parentable.__init__(self, TYPE, track_allocation=False) self._setparentstructure(parent, baseoffset_or_fieldname) # Keep the parent array alive, we share the same allocation. # Don't do it if we are inside a GC object, though -- it's someone @@ -1787,9 +1815,9 @@ else: initialization = 'malloc' if isinstance(T, Struct): - o = _struct(T, n, initialization=initialization) + o = _struct(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, Array): - o = _array(T, n, initialization=initialization) + o = _array(T, n, initialization=initialization, track_allocation=flavor == "raw" and not immortal) elif isinstance(T, OpaqueType): assert n is None o = _opaque(T, initialization=initialization) Modified: pypy/trunk/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rdict.py Thu Apr 29 20:37:15 2010 @@ -508,12 +508,14 @@ def ll_dict_lookup(d, key, hash): entries = d.entries + ENTRIES = lltype.typeOf(entries).TO + direct_compare = not hasattr(ENTRIES, 'no_direct_compare') mask = len(entries) - 1 i = hash & mask # do the first try before any looping if entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i # found the entry if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -548,7 +550,7 @@ return freeslot elif entries.valid(i): checkingkey = entries[i].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(i) == hash: # correct hash, maybe the key is e.g. a different pointer to Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Thu Apr 29 20:37:15 2010 @@ -48,6 +48,7 @@ result = isinstance(s_p, annmodel.SomePtr) return self.bookkeeper.immutablevalue(result) def specialize_call(self, hop): + hop.exception_cannot_occur() return hop.inputconst(lltype.Bool, hop.s_result.const) def llexternal(name, args, result, _callable=None, @@ -388,6 +389,7 @@ r_int_real = rarithmetic.build_int("r_int_real", r_int.SIGN, r_int.BITS, True) INT_real = lltype.build_number("INT", r_int_real) platform.numbertype_to_rclass[INT_real] = r_int_real +NUMBER_TYPES.append(INT_real) # ^^^ this creates at least the following names: # -------------------------------------------------------------------- Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Apr 29 20:37:15 2010 @@ -432,6 +432,29 @@ rffi.free_charp(p) assert not ALLOCATED # detects memory leaks in the test + def test_funcptr_cast(self): + eci = ExternalCompilationInfo( + separate_module_sources=[""" + long mul(long x, long y) { return x*y; } + long(*get_mul(long x)) () { return &mul; } + """], + export_symbols=['get_mul']) + get_mul = rffi.llexternal( + 'get_mul', [], + lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed)), + compilation_info=eci) + # This call returns a pointer to a function taking one argument + funcptr = get_mul() + # cast it to the "real" function type + FUNCTYPE2 = lltype.FuncType([lltype.Signed, lltype.Signed], + lltype.Signed) + cmul = rffi.cast(lltype.Ptr(FUNCTYPE2), funcptr) + # and it can be called with the expected number of arguments + res = cmul(41, 42) + assert res == 41 * 42 + raises(TypeError, cmul, 41) + raises(TypeError, cmul, 41, 42, 43) + def test_qsort(self): CMPFUNC = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], rffi.INT) qsort = rffi.llexternal('qsort', [rffi.VOIDP, Modified: pypy/trunk/pypy/tool/fixeol ============================================================================== --- pypy/trunk/pypy/tool/fixeol (original) +++ pypy/trunk/pypy/tool/fixeol Thu Apr 29 20:37:15 2010 @@ -48,7 +48,7 @@ return True def checkeolfile(path): - return path.ext in ('.txt', '.py', '.asc') + return path.ext in ('.txt', '.py', '.asc', '.h', '.c') def fixdirectory(path): print "+ checking directory", path, Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Apr 29 20:37:15 2010 @@ -1,10 +1,14 @@ import py -import sys, os +import sys, os, gc from pypy.translator.c.test import test_newgc from pypy.translator.translator import TranslationContext from pypy.translator.c.genc import CStandaloneBuilder from pypy.annotation.listdef import s_list_of_strings from pypy import conftest +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints +from pypy.rpython.lltypesystem.lloperation import llop class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved @@ -34,11 +38,14 @@ config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() + for f, inputtypes in cls.secondary_entrypoints: + a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() - cbuilder = CStandaloneBuilder(t, main, config=config) + cbuilder = CStandaloneBuilder(t, main, config=config, + secondary_entrypoints=cls.secondary_entrypoints) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) @@ -53,7 +60,12 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') + if config.translation.shared and os.name == 'posix': + env = 'LD_LIBRARY_PATH="%s" ' % (exe_name.dirpath(),) + else: + env = '' + g = os.popen( + '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) @@ -101,6 +113,7 @@ test_newgc.TestSemiSpaceGC): # for the individual tests see # ====> ../../test/test_newgc.py + secondary_entrypoints = [] def define_large_function(cls): class A(object): @@ -123,11 +136,6 @@ assert res == 1000 def define_callback_simple(cls): - import gc - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.rpython.annlowlevel import llhelper - from pypy.translator.tool.cbuild import ExternalCompilationInfo - c_source = py.code.Source(""" int mystuff(int(*cb)(int, int)) { @@ -153,11 +161,44 @@ return f - def test_callback_simple(self): res = self.run('callback_simple') assert res == 4900 + def define_secondary_entrypoint_callback(self): + @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') + def mycallback(a, b): + llop.gc_stack_bottom(lltype.Void) + rffi.stackcounter.stacks_counter += 1 + gc.collect() + rffi.stackcounter.stacks_counter -= 1 + return a + b + + c_source = py.code.Source(""" + int mystuff2() + { + return callback(40, 2) + callback(3, 4); + } + """) + + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + z = rffi.llexternal('mystuff2', [], lltype.Signed, + compilation_info=eci) + S = lltype.GcStruct('S', ('x', lltype.Signed)) + + self.secondary_entrypoints.extend(secondary_entrypoints["x42"]) + + def f(): + p = lltype.malloc(S) + p.x = 100 + result = z() + return result * p.x + + return f + + def test_secondary_entrypoint_callback(self): + res = self.run('secondary_entrypoint_callback') + assert res == 4900 class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): # for the individual tests see @@ -186,6 +227,13 @@ def define_callback_with_collect(cls): return lambda: 0 +class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC): + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.shared = True + return config + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Thu Apr 29 20:37:15 2010 @@ -1640,6 +1640,7 @@ format = 'mingw32' else: format = 'elf' + entrypoint = 'main' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1653,6 +1654,9 @@ elif sys.argv[1].startswith('-f'): format = sys.argv[1][2:] del sys.argv[1] + elif sys.argv[1].startswith('-m'): + entrypoint = sys.argv[1][2:] + del sys.argv[1] else: break tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) @@ -1669,7 +1673,7 @@ lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: - tracker.process(f, g, filename=fn) + tracker.process(f, g, entrypoint=entrypoint, filename=fn) except: g.close() os.unlink(lblfn) Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Thu Apr 29 20:37:15 2010 @@ -13,6 +13,7 @@ from pypy.translator.c.support import log, c_string_constant from pypy.rpython.typesystem import getfunctionptr from pypy.translator.c import gc +from pypy.rlib import exports def import_module_from_directory(dir, modname): file, pathname, description = imp.find_module(modname, [str(dir)]) @@ -77,14 +78,22 @@ self.outputfilename = outputfilename self.profbased = profbased - def _build(self, eci=ExternalCompilationInfo()): + def _build(self, eci=ExternalCompilationInfo(), shared=False): + outputfilename = self.outputfilename + if shared: + if outputfilename: + basename = outputfilename + else: + basename = self.cfiles[0].purebasename + outputfilename = 'lib' + basename return self.platform.compile(self.cfiles, self.eci.merge(eci), - outputfilename=self.outputfilename) + outputfilename=outputfilename, + standalone=not shared) - def build(self): + def build(self, shared=False): if self.profbased: return self._do_profbased() - return self._build() + return self._build(shared=shared) def _do_profbased(self): ProfDriver, args = self.profbased @@ -103,7 +112,8 @@ modulename = None split = False - def __init__(self, translator, entrypoint, config, gcpolicy=None): + def __init__(self, translator, entrypoint, config, gcpolicy=None, + secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) @@ -113,6 +123,7 @@ if gcpolicy is not None and gcpolicy.requires_stackless: config.translation.stackless = True self.eci = self.get_eci() + self.secondary_entrypoints = secondary_entrypoints def get_eci(self): pypy_include_dir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -159,7 +170,16 @@ self.c_entrypoint_name = None else: pfname = db.get(pf) + + for func, _ in self.secondary_entrypoints: + bk = translator.annotator.bookkeeper + db.get(getfunctionptr(bk.getdesc(func).getuniquegraph())) + self.c_entrypoint_name = pfname + + for obj in exports.EXPORTS_obj2name.keys(): + db.getcontainernode(obj) + exports.clear() db.complete() self.collect_compilation_info(db) @@ -240,6 +260,10 @@ if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 + if self.config.translation.shared: + defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup" + self.eci = self.eci.merge(ExternalCompilationInfo( + export_symbols=["pypy_main_startup"])) self.eci, cfile, extra = gen_source_standalone(db, modulename, targetdir, self.eci, @@ -404,12 +428,13 @@ if isinstance(self._module, isolate.Isolate): isolate.close_isolate(self._module) - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): pass class CStandaloneBuilder(CBuilder): standalone = True executable_name = None + shared_library_name = None def getprofbased(self): profbased = None @@ -450,9 +475,40 @@ return res.out, res.err return res.out - def compile(self): + def build_main_for_shared(self, shared_library_name, entrypoint, exe_name): + import time + time.sleep(1) + self.shared_library_name = shared_library_name + # build main program + eci = self.get_eci() + kw = {} + if self.translator.platform.so_ext == 'so': + kw['libraries'] = [self.shared_library_name.purebasename[3:]] + kw['library_dirs'] = [self.targetdir] + else: + kw['libraries'] = [self.shared_library_name.new(ext='')] + eci = eci.merge(ExternalCompilationInfo( + separate_module_sources=[''' + int %s(int argc, char* argv[]); + + int main(int argc, char* argv[]) + { return %s(argc, argv); } + ''' % (entrypoint, entrypoint) + ], + **kw + )) + eci = eci.convert_sources_to_files( + cache_dir=self.targetdir) + return self.translator.platform.compile( + [], eci, + outputfilename=exe_name) + + def compile(self, exe_name=None): assert self.c_source_filename assert not self._compiled + + shared = self.config.translation.shared + if (self.config.translation.gcrootfinder == "asmgcc" or self.config.translation.force_make): extra_opts = [] @@ -463,16 +519,23 @@ else: compiler = CCompilerDriver(self.translator.platform, [self.c_source_filename] + self.extrafiles, - self.eci, profbased=self.getprofbased()) - self.executable_name = compiler.build() + self.eci, profbased=self.getprofbased(), + outputfilename=exe_name) + self.executable_name = compiler.build(shared=shared) + if shared: + self.executable_name = self.build_main_for_shared( + self.executable_name, "pypy_main_startup", exe_name) assert self.executable_name self._compiled = True return self.executable_name - def gen_makefile(self, targetdir): + def gen_makefile(self, targetdir, exe_name=None): cfiles = [self.c_source_filename] + self.extrafiles - mk = self.translator.platform.gen_makefile(cfiles, self.eci, - path=targetdir) + mk = self.translator.platform.gen_makefile( + cfiles, self.eci, + path=targetdir, exe_name=exe_name, + shared=self.config.translation.shared) + if self.has_profopt(): profopt = self.config.translation.profopt mk.definition('ABS_TARGET', '$(shell python -c "import sys,os; print os.path.abspath(sys.argv[1])" $(TARGET))') @@ -516,6 +579,11 @@ mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') + if self.config.translation.shared: + mk.definition('PYPY_MAIN_FUNCTION', "pypy_main_startup") + else: + mk.definition('PYPY_MAIN_FUNCTION', "main") + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: @@ -541,7 +609,7 @@ 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -m$(PYPY_MAIN_FUNCTION) -t $*.s > $@'] ) mk.rule('gcmaptable.c', '$(GCMAPFILES)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') @@ -550,7 +618,7 @@ mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -m$(PYPY_MAIN_FUNCTION) -t $< > $*.gcmap') mk.rule('gcmaptable.s', '$(GCMAPFILES)', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Thu Apr 29 20:37:15 2010 @@ -10,6 +10,7 @@ from pypy.translator.c.support import cdecl, forward_cdecl, somelettersfrom from pypy.translator.c.support import c_char_array_constant, barebonearray from pypy.translator.c.primitive import PrimitiveType, name_signed +from pypy.rlib import exports from pypy.rlib.rarithmetic import isinf, isnan from pypy.rlib.rstackovf import _StackOverflow from pypy.translator.c import extfunc @@ -468,7 +469,7 @@ if USESLOTS: __slots__ = """db T obj typename implementationtypename - name ptrname + name ptrname compilation_info globalcontainer""".split() def __init__(self, db, T, obj): @@ -479,7 +480,10 @@ self.typename = db.gettype(T) #, who_asks=self) self.implementationtypename = db.gettype(T, varlength=self.getlength()) parent, parentindex = parentlink(obj) - if parent is None: + if obj in exports.EXPORTS_obj2name: + self.name = exports.EXPORTS_obj2name[obj] + self.globalcontainer = True + elif parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) self.globalcontainer = True else: @@ -490,6 +494,7 @@ if self.typename != self.implementationtypename: if db.gettypedefnode(T).extra_union_for_varlength: self.name += '.b' + self.compilation_info = getattr(obj, '_compilation_info', None) self.ptrname = '(&%s)' % self.name def is_thread_local(self): @@ -922,6 +927,8 @@ else: assert fnobj.external == 'CPython' return [CExternalFunctionCodeGenerator(fnobj, db)] + elif hasattr(fnobj._callable, "c_name"): + return [] else: raise ValueError, "don't know how to generate code for %r" % (fnobj,) Modified: pypy/trunk/pypy/translator/c/src/commondefs.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/commondefs.h (original) +++ pypy/trunk/pypy/translator/c/src/commondefs.h Thu Apr 29 20:37:15 2010 @@ -15,6 +15,13 @@ #include +#ifndef LLONG_MAX +#define LLONG_MAX __LONG_LONG_MAX__ +#endif +#ifndef LLONG_MIN +#define LLONG_MIN (-LLONG_MAX - 1LL) +#endif + #if INT_MAX != 2147483647 # error "unsupported value for INT_MAX" #endif @@ -62,9 +69,6 @@ /********************************************************/ -typedef long Py_intptr_t; -typedef unsigned long Py_uintptr_t; - #if ((-1) >> 1) > 0 # define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) \ ((I) < 0 ? -1-((-1-(I)) >> (J)) : (I) >> (J)) Modified: pypy/trunk/pypy/translator/c/src/main.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/main.h (original) +++ pypy/trunk/pypy/translator/c/src/main.h Thu Apr 29 20:37:15 2010 @@ -15,11 +15,15 @@ #ifndef PYPY_NOT_MAIN_FILE +#ifndef PYPY_MAIN_FUNCTION +#define PYPY_MAIN_FUNCTION main +#endif + #ifdef MS_WINDOWS #include "src/winstuff.c" #endif -int main(int argc, char *argv[]) +int PYPY_MAIN_FUNCTION(int argc, char *argv[]) { char *errmsg; int i, exitcode; Modified: pypy/trunk/pypy/translator/c/src/obmalloc.c ============================================================================== --- pypy/trunk/pypy/translator/c/src/obmalloc.c (original) +++ pypy/trunk/pypy/translator/c/src/obmalloc.c Thu Apr 29 20:37:15 2010 @@ -225,7 +225,7 @@ #define ulong unsigned long /* assuming >= 32 bits */ #undef uptr -#define uptr Py_uintptr_t +#define uptr unsigned long /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; @@ -439,7 +439,7 @@ arenabase = bp; nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * nfreepools == ARENA_SIZE); - excess = (uint) ((Py_uintptr_t)bp & POOL_SIZE_MASK); + excess = (uint) ((uint)bp & POOL_SIZE_MASK); if (excess != 0) { --nfreepools; arenabase += POOL_SIZE - excess; Modified: pypy/trunk/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_genc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_genc.py Thu Apr 29 20:37:15 2010 @@ -12,6 +12,7 @@ from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation +from pypy.rlib.entrypoint import entrypoint def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): @@ -396,3 +397,85 @@ if py.test.config.option.view: t.view() assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read() + +def test_entrypoints(): + def f(): + return 3 + + key = "test_entrypoints42" + @entrypoint(key, [int], "foobar") + def g(x): + return x + 42 + + t = Translation(f, [], backend="c", secondaryentrypoints="test_entrypoints42") + t.annotate() + compiled_fn = t.compile_c() + if py.test.config.option.view: + t.view() + assert 'foobar' in t.driver.cbuilder.c_source_filename.read() + +def test_exportstruct(): + from pypy.rlib.exports import export_struct + def f(): + return 42 + FOO = Struct("FOO", ("field1", Signed)) + foo = malloc(FOO, flavor="raw") + foo.field1 = 43 + export_struct("BarStruct", foo._obj) + t = Translation(f, [], backend="c") + t.annotate() + compiled_fn = t.compile_c() + if py.test.config.option.view: + t.view() + assert ' BarStruct ' in t.driver.cbuilder.c_source_filename.read() + +def test_recursive_llhelper(): + from pypy.rpython.annlowlevel import llhelper + from pypy.rpython.lltypesystem import lltype + from pypy.rlib.objectmodel import specialize + from pypy.rlib.nonconst import NonConstant + FT = lltype.ForwardReference() + FTPTR = lltype.Ptr(FT) + STRUCT = lltype.Struct("foo", ("bar", FTPTR)) + FT.become(lltype.FuncType([lltype.Ptr(STRUCT)], lltype.Signed)) + + class A: + def __init__(self, func, name): + self.func = func + self.name = name + def _freeze_(self): + return True + @specialize.memo() + def make_func(self): + f = getattr(self, "_f", None) + if f is not None: + return f + f = lambda *args: self.func(*args) + f.c_name = self.name + f.relax_sig_check = True + f.__name__ = "WRAP%s" % (self.name, ) + self._f = f + return f + def get_llhelper(self): + return llhelper(FTPTR, self.make_func()) + def f(s): + if s.bar == t.bar: + lltype.free(s, flavor="raw") + return 1 + lltype.free(s, flavor="raw") + return 0 + def g(x): + return 42 + def chooser(x): + s = lltype.malloc(STRUCT, flavor="raw") + if x: + s.bar = llhelper(FTPTR, a_f.make_func()) + else: + s.bar = llhelper(FTPTR, a_g.make_func()) + return f(s) + a_f = A(f, "f") + a_g = A(g, "g") + t = lltype.malloc(STRUCT, flavor="raw") + t.bar = llhelper(FTPTR, a_f.make_func()) + fn = compile(chooser, [bool]) + assert fn(True) Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Thu Apr 29 20:37:15 2010 @@ -66,6 +66,12 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue + keyword = conftest.option.keyword + if keyword: + if keyword.startswith('test_'): + keyword = keyword[len('test_'):] + if keyword not in fullname: + continue prefix, name = fullname.split('_', 1) definefunc = getattr(cls, fullname) func = definefunc.im_func(cls) Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_standalone.py (original) +++ pypy/trunk/pypy/translator/c/test/test_standalone.py Thu Apr 29 20:37:15 2010 @@ -16,11 +16,13 @@ class StandaloneTests(object): config = None - def compile(self, entry_point, debug=True): + def compile(self, entry_point, debug=True, shared=False): t = TranslationContext(self.config) t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() + t.config.translation.shared = shared + cbuilder = CStandaloneBuilder(t, entry_point, t.config) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) @@ -587,6 +589,19 @@ # The traceback stops at f() because it's the first function that # captures the AssertionError, which makes the program abort. + def test_shared(self, monkeypatch): + def f(argv): + print len(argv) + def entry_point(argv): + f(argv) + return 0 + t, cbuilder = self.compile(entry_point, shared=True) + assert cbuilder.shared_library_name is not None + assert cbuilder.shared_library_name != cbuilder.executable_name + monkeypatch.setenv('LD_LIBRARY_PATH', + cbuilder.shared_library_name.dirpath()) + out, err = cbuilder.cmdexec("a b") + assert out == "3" class TestMaemo(TestStandalone): def setup_class(cls): Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Thu Apr 29 20:37:15 2010 @@ -12,6 +12,7 @@ import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS +from pypy.rlib.entrypoint import secondary_entrypoints import py from pypy.tool.ansi_print import ansi_log @@ -93,7 +94,7 @@ if setopts is not None: self.config.set(**setopts) - + self.exe_name = exe_name self.extmod_name = extmod_name @@ -210,12 +211,25 @@ self.entry_point = entry_point self.translator = translator self.libdef = None + self.secondary_entrypoints = [] + + if self.config.translation.secondaryentrypoints: + for key in self.config.translation.secondaryentrypoints.split(","): + try: + points = secondary_entrypoints[key] + except KeyError: + raise KeyError( + "Entrypoints not found. I only know the keys %r." % + (", ".join(secondary_entrypoints.keys()), )) + self.secondary_entrypoints.extend(points) self.translator.driver_instrument_result = self.instrument_result def setup_library(self, libdef, policy=None, extra={}, empty_translator=None): + """ Used by carbon python only. """ self.setup(None, None, policy, extra, empty_translator) self.libdef = libdef + self.secondary_entrypoints = libdef.functions def instrument_result(self, args): backend, ts = self.get_backend_and_type_system() @@ -304,23 +318,25 @@ annmodel.DEBUG = self.config.translation.debug annotator = translator.buildannotator(policy=policy) + if self.secondary_entrypoints is not None: + for func, inputtypes in self.secondary_entrypoints: + if inputtypes == Ellipsis: + continue + rettype = annotator.build_types(func, inputtypes, False) + if self.entry_point: s = annotator.build_types(self.entry_point, self.inputtypes) - - self.sanity_check_annotation() - if self.standalone and s.knowntype != int: - raise Exception("stand-alone program entry point must return an " - "int (and not, e.g., None or always raise an " - "exception).") - annotator.simplify() - return s + translator.entry_point_graph = annotator.bookkeeper.getdesc(self.entry_point).getuniquegraph() else: - assert self.libdef is not None - for func, inputtypes in self.libdef.functions: - annotator.build_types(func, inputtypes) - func.c_name = func.func_name - self.sanity_check_annotation() - annotator.simplify() + s = None + + self.sanity_check_annotation() + if self.entry_point and self.standalone and s.knowntype != int: + raise Exception("stand-alone program entry point must return an " + "int (and not, e.g., None or always raise an " + "exception).") + annotator.simplify() + # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") @@ -471,7 +487,8 @@ else: from pypy.translator.c.genc import CExtModuleBuilder as CBuilder cbuilder = CBuilder(self.translator, self.entry_point, - config=self.config) + config=self.config, + secondary_entrypoints=self.secondary_entrypoints) cbuilder.stackless = self.config.translation.stackless if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name @@ -521,6 +538,10 @@ exename = mkexename(self.c_entryp) newexename = self.compute_exe_name() shutil.copy(str(exename), str(newexename)) + if self.cbuilder.shared_library_name is not None: + soname = self.cbuilder.shared_library_name + newsoname = newexename.new(basename=soname.basename) + shutil.copy(str(soname), str(newsoname)) self.c_entryp = newexename self.log.info("created: %s" % (self.c_entryp,)) @@ -529,7 +550,10 @@ translator/platform """ cbuilder = self.cbuilder - cbuilder.compile() + kwds = {} + if self.standalone: + kwds['exe_name'] = self.compute_exe_name().basename + cbuilder.compile(**kwds) if self.standalone: self.c_entryp = cbuilder.executable_name Modified: pypy/trunk/pypy/translator/goal/ann_override.py ============================================================================== --- pypy/trunk/pypy/translator/goal/ann_override.py (original) +++ pypy/trunk/pypy/translator/goal/ann_override.py Thu Apr 29 20:37:15 2010 @@ -81,7 +81,7 @@ def attach_lookup(pol, t, attr): cached = "cached_%s" % attr - if not t.is_heaptype(): + if not t.is_heaptype() and not t.is_cpytype(): pol._remember_immutable(t, cached) setattr(t, cached, t._lookup(attr)) return True @@ -89,7 +89,7 @@ def attach_lookup_in_type_where(pol, t, attr): cached = "cached_where_%s" % attr - if not t.is_heaptype(): + if not t.is_heaptype() and not t.is_cpytype(): pol._remember_immutable(t, cached) setattr(t, cached, t._lookup_where(attr)) return True @@ -177,14 +177,14 @@ CACHED_LOOKUP = """ def lookup_%(attr)s(space, w_obj, name): w_type = space.type(w_obj) - if not w_type.is_heaptype(): + if not w_type.is_heaptype() and not w_type.is_cpytype(): return w_type.cached_%(attr)s return w_type.lookup("%(attr)s") """ CACHED_LOOKUP_IN_TYPE_WHERE = """ def lookup_in_type_where_%(attr)s(space, w_type, name): - if not w_type.is_heaptype(): + if not w_type.is_heaptype() and not w_type.is_cpytype(): return w_type.cached_where_%(attr)s return w_type.lookup_where("%(attr)s") """ Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Thu Apr 29 20:37:15 2010 @@ -70,7 +70,8 @@ env) return ExecutionResult(returncode, stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): raise NotImplementedError("Pure abstract baseclass") def __repr__(self): @@ -105,10 +106,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() - for line in stderrlines[:50]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 50: - log.ERROR('...') raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Thu Apr 29 20:37:15 2010 @@ -18,7 +18,7 @@ self.cc = cc def _args_for_shared(self, args): - return (self.shared_only + ['-bundle', '-undefined', 'dynamic_lookup'] + return (self.shared_only + ['-dynamiclib', '-undefined', 'dynamic_lookup'] + args) def include_dirs_for_libffi(self): Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Thu Apr 29 20:37:15 2010 @@ -7,7 +7,7 @@ name = "linux" link_flags = ['-pthread', '-lrt'] - cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall'] + cflags = ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] standalone_only = [] shared_only = ['-fPIC'] so_ext = 'so' Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Thu Apr 29 20:37:15 2010 @@ -36,6 +36,10 @@ cwd=str(cfile.dirpath())) return oname + def _link_args_from_eci(self, eci, standalone): + eci = eci.convert_exportsymbols_to_file() + return Platform._link_args_from_eci(self, eci, standalone) + def _link(self, cc, ofiles, link_args, standalone, exe_name): args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] @@ -55,7 +59,9 @@ # strip compiler flags return [entry[2:] for entry in out.split()] - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): + eci = eci.convert_exportsymbols_to_file() cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -67,6 +73,16 @@ if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + + if shared: + libname = exe_name.new(ext='').basename + target_name = 'lib' + exe_name.new(ext=self.so_ext).basename + else: + target_name = exe_name.basename + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci @@ -88,8 +104,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -97,7 +113,7 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), @@ -115,6 +131,16 @@ for rule in rules: m.rule(*rule) + if shared: + m.definition('SHARED_IMPORT_LIB', libname), + m.rule('main.c', '', + 'echo "' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], + '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Thu Apr 29 20:37:15 2010 @@ -186,7 +186,8 @@ raise CompilationError(stdout, stderr) - def gen_makefile(self, cfiles, eci, exe_name=None, path=None): + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, + shared=False): cfiles = [py.path.local(f) for f in cfiles] cfiles += [py.path.local(f) for f in eci.separate_module_files] @@ -202,6 +203,17 @@ m.exe_name = exe_name m.eci = eci + linkflags = self.link_flags + if shared: + linkflags = self._args_for_shared(linkflags) + [ + '/EXPORT:$(PYPY_MAIN_FUNCTION)'] + + if shared: + so_name = exe_name.new(ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + def pypyrel(fpath): rel = py.path.local(fpath).relto(pypypath) if rel: @@ -218,8 +230,8 @@ m.comment('automatically generated makefile') definitions = [ ('PYPYDIR', autopath.pypydir), - ('TARGET', exe_name.basename), - ('DEFAULT_TARGET', '$(TARGET)'), + ('TARGET', target_name), + ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), ('OBJECTS', rel_ofiles), ('LIBS', self._libs(eci.libraries)), @@ -227,12 +239,13 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags), ('CFLAGSEXTRA', list(eci.compile_extra)), - ('LDFLAGS', self.link_flags), + ('LDFLAGS', linkflags), ('LDFLAGSEXTRA', list(eci.link_extra)), ('CC', self.cc), ('CC_LINK', self.link), ('MASM', self.masm), ] + for args in definitions: m.definition(*args) @@ -253,6 +266,16 @@ 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) + if shared: + m.definition('SHARED_IMPORT_LIB', so_name.new(ext='lib').basename), + m.rule('main.c', '', + 'echo ' + 'int $(PYPY_MAIN_FUNCTION)(int, char*[]); ' + 'int main(int argc, char* argv[]) ' + '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); } > $@') + m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.obj'], + '$(CC_LINK) /nologo main.obj $(SHARED_IMPORT_LIB) /out:$@') + return m def execute_makefile(self, path_to_makefile, extra_opts=[]): Modified: pypy/trunk/pypy/translator/test/test_unsimplify.py ============================================================================== --- pypy/trunk/pypy/translator/test/test_unsimplify.py (original) +++ pypy/trunk/pypy/translator/test/test_unsimplify.py Thu Apr 29 20:37:15 2010 @@ -9,6 +9,7 @@ def translate(func, argtypes, type_system="lltype"): t = TranslationContext() t.buildannotator().build_types(func, argtypes) + t.entry_point_graph = graphof(t, func) t.buildrtyper(type_system=type_system).specialize() return graphof(t, func), t Modified: pypy/trunk/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/cbuild.py Thu Apr 29 20:37:15 2010 @@ -260,24 +260,49 @@ d['separate_module_files'] += tuple(files) return ExternalCompilationInfo(**d) + def convert_exportsymbols_to_file(self): + if not self.export_symbols: + return self + num = 0 + while 1: + file_name = udir.join('dynamic-symbols-%i' % num) + num += 1 + if not file_name.check(): + break + + f = file_name.open("w") + f.write("{\n") + for sym in self.export_symbols: + f.write("%s;\n" % (sym,)) + f.write("};") + f.close() + d = self._copy_attributes() + d['link_extra'] += ("-Wl,--dynamic-list=" + str(file_name), ) + d['export_symbols'] = () + return ExternalCompilationInfo(**d) + + def get_module_files(self): d = self._copy_attributes() files = d['separate_module_files'] d['separate_module_files'] = () return files, ExternalCompilationInfo(**d) - def compile_shared_lib(self): + def compile_shared_lib(self, outputfilename=None): self = self.convert_sources_to_files() if not self.separate_module_files: return self - # find more or less unique name there - basepath = py.path.local(self.separate_module_files[0]).dirpath() - pth = basepath.join('externmod').new(ext=host.so_ext) - num = 0 - while pth.check(): - pth = basepath.join('externmod_%d' % (num,)).new(ext=host.so_ext) - num += 1 - lib = str(host.compile([], self, outputfilename=pth.purebasename, + if outputfilename is None: + # find more or less unique name there + basepath = py.path.local(self.separate_module_files[0]).dirpath() + pth = basepath.join('externmod').new(ext=host.so_ext) + num = 0 + while pth.check(): + pth = basepath.join( + 'externmod_%d' % (num,)).new(ext=host.so_ext) + num += 1 + outputfilename=pth.purebasename + lib = str(host.compile([], self, outputfilename=outputfilename, standalone=False)) d = self._copy_attributes() d['libraries'] += (lib,) Modified: pypy/trunk/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/trunk/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/trunk/pypy/translator/tool/test/test_cbuild.py Thu Apr 29 20:37:15 2010 @@ -71,6 +71,17 @@ e = ExternalCompilationInfo() assert e.convert_sources_to_files() is e + def test_convert_sources_to_c_files(self): + eci = ExternalCompilationInfo( + export_symbols=("foo", ) + ) + neweci = eci.convert_exportsymbols_to_file() + le = neweci.link_extra[-1] + assert "foo;" in file(le.rsplit("=", 1)[1]).read() + e = ExternalCompilationInfo() + assert e.convert_exportsymbols_to_file() is e + + def test_make_shared_lib(self): eci = ExternalCompilationInfo( separate_module_sources = [''' Modified: pypy/trunk/pypy/translator/unsimplify.py ============================================================================== --- pypy/trunk/pypy/translator/unsimplify.py (original) +++ pypy/trunk/pypy/translator/unsimplify.py Thu Apr 29 20:37:15 2010 @@ -168,7 +168,7 @@ if own_annhelper: annhelper.finish() - entry_point = translator.graphs[0] + entry_point = translator.entry_point_graph v = copyvar(translator.annotator, entry_point.getreturnvar()) extrablock = Block([v]) v_none = varoftype(lltype.Void) From fijal at codespeak.net Thu Apr 29 20:39:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:39:35 +0200 (CEST) Subject: [pypy-svn] r74237 - pypy/trunk/pypy/doc/config Message-ID: <20100429183935.D79ED282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:39:34 2010 New Revision: 74237 Added: pypy/trunk/pypy/doc/config/translation.secondaryentrypoints.txt (contents, props changed) Log: Add some documentation for this option Added: pypy/trunk/pypy/doc/config/translation.secondaryentrypoints.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/translation.secondaryentrypoints.txt Thu Apr 29 20:39:34 2010 @@ -0,0 +1 @@ +Enable secondary entrypoints support list. Needed for cpyext module. From fijal at codespeak.net Thu Apr 29 20:40:46 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:40:46 +0200 (CEST) Subject: [pypy-svn] r74238 - pypy/branch/cpython-extension Message-ID: <20100429184046.E93B9282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:40:45 2010 New Revision: 74238 Removed: pypy/branch/cpython-extension/ Log: Remove merged branch From fijal at codespeak.net Thu Apr 29 20:41:28 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:41:28 +0200 (CEST) Subject: [pypy-svn] r74239 - pypy/branch/cpyext-remove-pyolifeline Message-ID: <20100429184128.10025282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:41:26 2010 New Revision: 74239 Added: pypy/branch/cpyext-remove-pyolifeline/ (props changed) - copied from r74238, pypy/trunk/ Log: A branch to remove pyolifeline from W_Root object From fijal at codespeak.net Thu Apr 29 20:44:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 20:44:23 +0200 (CEST) Subject: [pypy-svn] r74240 - pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/test/zfec Message-ID: <20100429184423.92F91282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 20:44:22 2010 New Revision: 74240 Removed: pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/test/zfec/ Log: I'm pretty sure I did not want to have that in From fijal at codespeak.net Thu Apr 29 21:01:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 21:01:22 +0200 (CEST) Subject: [pypy-svn] r74241 - pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext Message-ID: <20100429190122.DA076282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 21:01:18 2010 New Revision: 74241 Modified: pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/typeobject.py Log: use RWeakKeyDictionary instead of pyolifeline, works so far Modified: pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/cpyext-remove-pyolifeline/pypy/module/cpyext/typeobject.py Thu Apr 29 21:01:18 2010 @@ -2,8 +2,8 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype -from pypy.tool.pairtype import extendabletype from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch @@ -199,15 +199,6 @@ if not pto.c_tp_new: pto.c_tp_new = base_pto.c_tp_new -class __extend__(W_Root): - __metaclass__ = extendabletype - __slots__ = ("_pyolifeline", ) # hint for the annotator - _pyolifeline = None - def set_pyolifeline(self, lifeline): - self._pyolifeline = lifeline - def get_pyolifeline(self): - return self._pyolifeline - class PyOLifeline(object): def __init__(self, space, pyo): self.pyo = pyo @@ -220,6 +211,8 @@ self.pyo = lltype.nullptr(PyObject.TO) # XXX handle borrowed objects here +lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) + def check_descr(space, w_self, pto): w_type = from_ref(space, (rffi.cast(PyObject, pto))) if not space.is_true(space.isinstance(w_self, w_type)): @@ -392,7 +385,7 @@ # extension modules def pyctype_make_ref(space, w_type, w_obj, itemcount=0): - lifeline = w_obj.get_pyolifeline() + lifeline = lifeline_dict.get(w_obj) if lifeline is not None: # make old PyObject ready for use in C code py_obj = lifeline.pyo assert py_obj.c_ob_refcnt == 0 @@ -400,7 +393,7 @@ else: typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) + lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) return py_obj @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, From fijal at codespeak.net Thu Apr 29 21:10:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 21:10:54 +0200 (CEST) Subject: [pypy-svn] r74242 - pypy/trunk/pypy/interpreter/test Message-ID: <20100429191054.04E89282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 21:10:26 2010 New Revision: 74242 Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py Log: Fix nameerror in the test, discover another error Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_typedef.py (original) +++ pypy/trunk/pypy/interpreter/test/test_typedef.py Thu Apr 29 21:10:26 2010 @@ -138,9 +138,10 @@ pass def fget(self, space, w_self): assert self is prop + prop = typedef.GetSetProperty(fget, use_closure=True) W_SomeType.typedef = typedef.TypeDef( 'some_type', - x=typedef.GetSetProperty(fget, use_closure=True)) + x=prop) w_obj = self.space.wrap(W_SomeType()) assert self.space.getattr(w_obj, self.space.wrap('x')) is None From fijal at codespeak.net Thu Apr 29 21:12:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 21:12:35 +0200 (CEST) Subject: [pypy-svn] r74243 - pypy/trunk/pypy/interpreter/test Message-ID: <20100429191235.3F459282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 21:11:58 2010 New Revision: 74243 Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py Log: fix the test Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_typedef.py (original) +++ pypy/trunk/pypy/interpreter/test/test_typedef.py Thu Apr 29 21:11:58 2010 @@ -143,7 +143,7 @@ 'some_type', x=prop) w_obj = self.space.wrap(W_SomeType()) - assert self.space.getattr(w_obj, self.space.wrap('x')) is None + assert self.space.getattr(w_obj, self.space.wrap('x')) is self.space.w_None class AppTestTypeDef: From fijal at codespeak.net Thu Apr 29 21:35:42 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 21:35:42 +0200 (CEST) Subject: [pypy-svn] r74244 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100429193542.F07CB282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 21:35:40 2010 New Revision: 74244 Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py Log: To be on the safe side, don't assume entry_point is graphs[0] Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Thu Apr 29 21:35:40 2010 @@ -231,7 +231,7 @@ def check_access_directly_sanity(self, graphs): from pypy.translator.backendopt.inline import collect_called_graphs jit_graphs = set(graphs) - for graph in collect_called_graphs(self.translator.graphs[0], + for graph in collect_called_graphs(self.translator.entry_point_graph, self.translator): if graph in jit_graphs: continue From fijal at codespeak.net Thu Apr 29 21:43:39 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 21:43:39 +0200 (CEST) Subject: [pypy-svn] r74245 - in pypy/trunk/pypy: jit/backend/test rpython/test Message-ID: <20100429194339.00285282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 21:43:38 2010 New Revision: 74245 Modified: pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/rpython/test/test_llinterp.py Log: Stick entry_point_graph here and there Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Thu Apr 29 21:43:38 2010 @@ -2,7 +2,7 @@ import sys from pypy.rlib.debug import debug_print from pypy.rlib.jit import OPTIMIZER_FULL -from pypy.translator.translator import TranslationContext +from pypy.translator.translator import TranslationContext, graphof class BaseCompiledMixin(object): @@ -58,6 +58,7 @@ t.buildannotator().build_types(function, [int] * len(args)) t.buildrtyper(type_system=self.type_system).specialize() + t.entry_point_graph = graphof(t, function) warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True, CPUClass=self.CPUClass, **kwds) Modified: pypy/trunk/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_llinterp.py (original) +++ pypy/trunk/pypy/rpython/test/test_llinterp.py Thu Apr 29 21:43:38 2010 @@ -62,6 +62,7 @@ t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) + t.entry_point_graph = graph return t, typer, graph _lastinterpreted = [] From afa at codespeak.net Thu Apr 29 22:02:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 22:02:58 +0200 (CEST) Subject: [pypy-svn] r74246 - pypy/trunk/pypy/rlib Message-ID: <20100429200258.E9433282BEF@codespeak.net> Author: afa Date: Thu Apr 29 22:02:56 2010 New Revision: 74246 Modified: pypy/trunk/pypy/rlib/rarithmetic.py Log: Attempt to fix many tests. Let's see if we broke something for cpyext Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Thu Apr 29 22:02:56 2010 @@ -127,6 +127,8 @@ def compute_restype(self_type, other_type): if self_type is other_type: + if self_type is bool: + return int return self_type if other_type in (bool, int, long): if self_type is bool: From fijal at codespeak.net Thu Apr 29 22:46:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 22:46:27 +0200 (CEST) Subject: [pypy-svn] r74247 - in pypy/trunk: . lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/command Message-ID: <20100429204627.BE48A282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 22:46:25 2010 New Revision: 74247 Modified: pypy/trunk/ (props changed) pypy/trunk/lib-python/modified-2.5.2/distutils/command/build_ext.py pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/trunk/lib-python/modified-2.5.2/distutils/spawn.py pypy/trunk/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Log: Oops, missing part of the merge of cpyext branch Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/command/build_ext.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/command/build_ext.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/command/build_ext.py Thu Apr 29 22:46:25 2010 @@ -167,7 +167,7 @@ # for Release and Debug builds. # also Python's library directory must be appended to library_dirs if os.name == 'nt': - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + self.library_dirs.append(os.path.join(sys.pypy_prefix, 'pypy', '_interfaces')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: @@ -175,8 +175,8 @@ # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) - self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) + #self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + #self.library_dirs.append(os.path.join(sys.exec_prefix, 'PCBuild')) # OS/2 (EMX) doesn't support Debug vs Release builds, but has the # import libraries in its "Config" subdirectory @@ -645,24 +645,12 @@ shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. + # The python library is always needed on Windows. if sys.platform == "win32": - from distutils.msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries + pythonlib = 'libpypy-c.exe' + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] elif sys.platform == "os2emx": # EMX/GCC requires the python library explicitly, and I # believe VACPP does as well (though not confirmed) - AIM Apr01 Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/msvccompiler.py Thu Apr 29 22:46:25 2010 @@ -1,154 +1,162 @@ -"""distutils.msvccompiler +"""distutils.msvc9compiler Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. """ # Written by Perry Stoll # hacked by Robin Becker and Thomas Heller to do a better job of # finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes -# This module should be kept compatible with Python 2.1. - -__revision__ = "$Id: msvccompiler.py 54645 2007-04-01 18:29:47Z neal.norwitz $" +__revision__ = "$Id: msvc9compiler.py 78713 2010-03-06 02:17:28Z tarek.ziade $" -import sys, os, string -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options +import os +import subprocess +import sys +import re + +from distutils.errors import (DistutilsExecError, DistutilsPlatformError, + CompileError, LibError, LinkError) +from distutils.ccompiler import CCompiler, gen_lib_options from distutils import log +from distutils.util import get_platform -_can_read_reg = 0 -try: - import _winreg - - _can_read_reg = 1 - hkey_mod = _winreg - - RegOpenKeyEx = _winreg.OpenKeyEx - RegEnumKey = _winreg.EnumKey - RegEnumValue = _winreg.EnumValue - RegError = _winreg.error +import _winreg -except ImportError: - try: - import win32api - import win32con - _can_read_reg = 1 - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules _winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) +RegOpenKeyEx = _winreg.OpenKeyEx +RegEnumKey = _winreg.EnumKey +RegEnumValue = _winreg.EnumValue +RegError = _winreg.error + +HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) + +VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" +VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" +WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" +NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targetting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', + 'win-ia64' : 'ia64', +} -def read_keys(base, key): - """Return list of registry keys.""" +class Reg: + """Helper class to read values from the registry + """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while 1: + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" try: - k = RegEnumKey(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - L.append(k) - i = i + 1 - return L + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) -def read_values(base, key): - """Return dict of registry keys and values. + def read_values(cls, base, key): + """Return dict of registry keys and values. - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while 1: + All names are converted to lowercase. + """ try: - name, value, type = RegEnumValue(handle, i) + handle = RegOpenKeyEx(base, key) except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i = i + 1 - return d - -def convert_mbcs(s): - enc = getattr(s, "encode", None) - if enc is not None: - try: - s = enc("mbcs") - except UnicodeError: - pass - return s + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) class MacroExpander: def __init__(self, version): self.macros = {} + self.vsbase = VS_BASE % version self.load_macros(version) def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break + self.macros["$(%s)" % macro] = Reg.get_value(path, key) def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError, exc: # - raise DistutilsPlatformError, \ - ("""Python was built with Visual Studio 2003; + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, +Visual Studio 2008 was not found on this system. If you have Cygwin installed, you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] def sub(self, s): for k, v in self.macros.items(): - s = string.replace(s, k, v) + s = s.replace(k, v) return s def get_build_version(): @@ -157,35 +165,7 @@ For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - - prefix = "MSC v." - i = string.find(sys.version, prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = string.find(sys.version, prefix) - if i == -1: - return "Intel" - j = string.find(sys.version, ")", i) - return sys.version[i+len(prefix):j] + return 8.0 def normalize_and_reduce_paths(paths): """Return a list of normalized paths with duplicates removed. @@ -201,8 +181,104 @@ reduced_paths.append(np) return reduced_paths +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + + # trying Express edition + if productdir is None: + vsbase = VSEXPRESS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + productdir = None + log.debug("Unable to find productdir in registry") + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = set(("include", "lib", "libpath", "path")) + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) -class MSVCCompiler (CCompiler) : +class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -232,27 +308,29 @@ static_lib_format = shared_lib_format = '%s%s' exe_extension = '.exe' - def __init__ (self, verbose=0, dry_run=0, force=0): + def __init__(self, verbose=0, dry_run=0, force=0): CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name self.initialized = False - def initialize(self): - self.__paths = [] - if os.environ.has_key("DISTUTILS_USE_SDK") and os.environ.has_key("MSSdk") and self.find_exe("cl.exe"): + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + ok_plats = 'win32', 'win-amd64', 'win-ia64' + if plat_name not in ok_plats: + raise DistutilsPlatformError("--plat-name must be one of %s" % + (ok_plats,)) + + if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): # Assume that the SDK set up everything alright; don't try to be # smarter self.cc = "cl.exe" @@ -261,36 +339,54 @@ self.rc = "rc.exe" self.mc = "mc.exe" else: - self.__paths = self.get_msvc_paths("path") + # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; + # to cross compile, you use 'x86_amd64'. + # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross + # compile use 'x86' (ie, it runs the x86 compiler directly) + # No idea how itanium handles this, if at all. + if plat_name == get_platform() or plat_name == 'win32': + # native build or cross-compile to win32 + plat_spec = PLAT_TO_VCVARS[plat_name] + else: + # cross compile from win32 -> some 64bit + plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ + PLAT_TO_VCVARS[plat_name] + + vc_env = query_vcvarsall(VERSION, plat_spec) + + # take care to only use strings in the environment. + self.__paths = vc_env['path'].encode('mbcs').split(os.pathsep) + os.environ['lib'] = vc_env['lib'].encode('mbcs') + os.environ['include'] = vc_env['include'].encode('mbcs') - if len (self.__paths) == 0: - raise DistutilsPlatformError, \ - ("Python was built with %s, " + if len(self.__paths) == 0: + raise DistutilsPlatformError("Python was built with %s, " "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." % self.__product) + "version of the compiler, but it isn't installed." + % self.__product) self.cc = self.find_exe("cl.exe") self.linker = self.find_exe("link.exe") self.lib = self.find_exe("lib.exe") self.rc = self.find_exe("rc.exe") # resource compiler self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') + #self.set_path_env_var('lib') + #self.set_path_env_var('include') # extend the MSVC path with the current path try: - for p in string.split(os.environ['path'], ';'): + for p in os.environ['path'].split(';'): self.__paths.append(p) except KeyError: pass self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = string.join(self.__paths, ';') + os.environ['path'] = ";".join(self.__paths) self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , + if self.__arch == "x86": + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/Z7', '/D_DEBUG'] else: # Win64 @@ -304,20 +400,16 @@ self.ldflags_shared_debug = [ '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] self.ldflags_static = [ '/nologo'] self.initialized = True # -- Worker methods ------------------------------------------------ - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): + def object_filenames(self, + source_filenames, + strip_dir=0, + output_dir=''): # Copied from ccompiler.py, extended to return .res as 'object'-file # for .rc input file if output_dir is None: output_dir = '' @@ -344,17 +436,16 @@ base + self.obj_extension)) return obj_names - # object_filenames () - def compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - if not self.initialized: self.initialize() - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info compile_opts = extra_preargs or [] compile_opts.append ('/c') @@ -383,13 +474,12 @@ input_opt = src output_opt = "/fo" + obj try: - self.spawn ([self.rc] + pp_opts + - [output_opt] + [input_opt]) + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. # * '-h dir' specifies the directory for the # generated include file @@ -401,99 +491,95 @@ # we use the source-directory for the include file and # the build directory for the RC file and message # resources. This works at least for win32all. - - h_dir = os.path.dirname (src) - rc_dir = os.path.dirname (obj) + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) try: # first compile .MC to .RC and .H file - self.spawn ([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) base, _ = os.path.splitext (os.path.basename (src)) rc_file = os.path.join (rc_dir, base + '.rc') # then compile .RC to .RES file - self.spawn ([self.rc] + - ["/fo" + obj] + [rc_file]) + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) continue else: # how to handle this file? - raise CompileError ( - "Don't know how to compile %s to %s" % \ - (src, obj)) + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) output_opt = "/Fo" + obj try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) except DistutilsExecError, msg: - raise CompileError, msg + raise CompileError(msg) return objects - # compile () - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=0, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) - if self._need_link (objects, output_filename): + if self._need_link(objects, output_filename): lib_args = objects + ['/OUT:' + output_filename] if debug: - pass # XXX what goes here? + pass # XXX what goes here? try: - self.spawn ([self.lib] + lib_args) + self.spawn([self.lib] + lib_args) except DistutilsExecError, msg: - raise LibError, msg - + raise LibError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # create_static_lib () - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) + def link(self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + (libraries, library_dirs, runtime_library_dirs) = fixed_args if runtime_library_dirs: self.warn ("I don't know what to do with 'runtime_library_dirs': " + str (runtime_library_dirs)) - lib_opts = gen_lib_options (self, - library_dirs, runtime_library_dirs, - libraries) + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): + output_filename = os.path.join(output_dir, output_filename) + if self._need_link(objects, output_filename): if target_desc == CCompiler.EXECUTABLE: if debug: ldflags = self.ldflags_shared_debug[1:] @@ -517,47 +603,99 @@ # needed! Make sure they are generated in the temporary build # directory. Since they have different names for debug and release # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) if export_symbols is not None: (dll_name, dll_ext) = os.path.splitext( os.path.basename(output_filename)) implib_file = os.path.join( - os.path.dirname(objects[0]), + build_temp, self.library_filename(dll_name)) ld_args.append ('/IMPLIB:' + implib_file) + # Embedded manifests are recommended - see MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can embed it later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + if extra_preargs: ld_args[:0] = extra_preargs if extra_postargs: ld_args.extend(extra_postargs) - self.mkpath (os.path.dirname (output_filename)) + self.mkpath(os.path.dirname(output_filename)) try: - self.spawn ([self.linker] + ld_args) + self.spawn([self.linker] + ld_args) except DistutilsExecError, msg: - raise LinkError, msg + raise LinkError(msg) + # embed the manifest + # XXX - this is somewhat fragile - if mt.exe fails, distutils + # will still consider the DLL up-to-date, but it will not have a + # manifest. Maybe we should link to a temp file? OTOH, that + # implies a build environment error that shouldn't go undetected. + if target_desc == CCompiler.EXECUTABLE: + mfid = 1 + else: + mfid = 2 + self._remove_visual_c_ref(temp_manifest) + out_arg = '-outputresource:%s;%s' % (output_filename, mfid) + try: + self.spawn(['mt.exe', '-nologo', '-manifest', + temp_manifest, out_arg]) + except DistutilsExecError, msg: + raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # link () - + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = "\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + finally: + manifest_f.close() + except IOError: + pass # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. - def library_dir_option (self, dir): + def library_dir_option(self, dir): return "/LIBPATH:" + dir - def runtime_library_dir_option (self, dir): - raise DistutilsPlatformError, \ - "don't know how to set runtime library search path for MSVC++" + def runtime_library_dir_option(self, dir): + raise DistutilsPlatformError( + "don't know how to set runtime library search path for MSVC++") - def library_option (self, lib): - return self.library_filename (lib) + def library_option(self, lib): + return self.library_filename(lib) - def find_library_file (self, dirs, lib, debug=0): + def find_library_file(self, dirs, lib, debug=0): # Prefer a debugging library if found (and requested), but deal # with it if we don't have one. if debug: @@ -573,8 +711,6 @@ # Oops, didn't find it in *any* of 'dirs' return None - # find_library_file () - # Helper methods for using the MSVC registry settings def find_exe(self, exe): @@ -586,67 +722,15 @@ absolute path that is known to exist. If none of them work, just return the original program name, 'exe'. """ - for p in self.__paths: fn = os.path.join(os.path.abspath(p), exe) if os.path.isfile(fn): return fn # didn't find it; try existing path - for p in string.split(os.environ['Path'],';'): + for p in os.environ['Path'].split(';'): fn = os.path.join(os.path.abspath(p),exe) if os.path.isfile(fn): return fn return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return string.split(self.__macros.sub(d[path]), ";") - else: - return string.split(d[path], ";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = string.join(p, ';') Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/spawn.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/spawn.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/spawn.py Thu Apr 29 22:46:25 2010 @@ -69,7 +69,6 @@ dry_run=0): executable = cmd[0] - cmd = _nt_quote_args(cmd) if search_path: # either we find one or it stays the same executable = find_executable(executable) or executable @@ -77,7 +76,8 @@ if not dry_run: # spawn for NT requires a full path to the .exe try: - rc = os.spawnv(os.P_WAIT, executable, cmd) + import subprocess + rc = subprocess.call(cmd) except OSError, exc: # this seems to happen when the command isn't found raise DistutilsExecError, \ Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py Thu Apr 29 22:46:25 2010 @@ -11,6 +11,12 @@ python_build = False +def get_python_inc(plat_specific=0, prefix=None): + from os.path import join as j + if plat_specific: + return j(sys.pypy_prefix, "pypy", "_interfaces") + return j(sys.pypy_prefix, 'pypy', 'module', 'cpyext', 'include') + def get_python_version(): """Return a string containing the major and minor Python version, leaving off the patchlevel. Sample return values could be '1.5' @@ -47,6 +53,7 @@ """Initialize the module as appropriate for POSIX systems.""" g = {} g['EXE'] = "" + g['SO'] = ".so" global _config_vars _config_vars = g @@ -56,6 +63,7 @@ """Initialize the module as appropriate for NT""" g = {} g['EXE'] = ".exe" + g['SO'] = ".pyd" global _config_vars _config_vars = g From fijal at codespeak.net Thu Apr 29 22:52:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 22:52:00 +0200 (CEST) Subject: [pypy-svn] r74248 - in pypy/trunk: . pypy/module/cpyext pypy/module/cpyext/test/zfec Message-ID: <20100429205200.5A18A282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 22:51:58 2010 New Revision: 74248 Removed: pypy/trunk/pypy/module/cpyext/test/zfec/ Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/module/cpyext/typeobject.py Log: Merge the cpyext-remove-pyolifeline branch. This branch removes _pyolifeline on all W_Root objects and instead uses RWeakKeyDictionary Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Thu Apr 29 22:51:58 2010 @@ -2,8 +2,8 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype -from pypy.tool.pairtype import extendabletype from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch @@ -199,15 +199,6 @@ if not pto.c_tp_new: pto.c_tp_new = base_pto.c_tp_new -class __extend__(W_Root): - __metaclass__ = extendabletype - __slots__ = ("_pyolifeline", ) # hint for the annotator - _pyolifeline = None - def set_pyolifeline(self, lifeline): - self._pyolifeline = lifeline - def get_pyolifeline(self): - return self._pyolifeline - class PyOLifeline(object): def __init__(self, space, pyo): self.pyo = pyo @@ -220,6 +211,8 @@ self.pyo = lltype.nullptr(PyObject.TO) # XXX handle borrowed objects here +lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) + def check_descr(space, w_self, pto): w_type = from_ref(space, (rffi.cast(PyObject, pto))) if not space.is_true(space.isinstance(w_self, w_type)): @@ -392,7 +385,7 @@ # extension modules def pyctype_make_ref(space, w_type, w_obj, itemcount=0): - lifeline = w_obj.get_pyolifeline() + lifeline = lifeline_dict.get(w_obj) if lifeline is not None: # make old PyObject ready for use in C code py_obj = lifeline.pyo assert py_obj.c_ob_refcnt == 0 @@ -400,7 +393,7 @@ else: typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - w_obj.set_pyolifeline(PyOLifeline(space, py_obj)) + lifeline_dict.set(w_obj, PyOLifeline(space, py_obj)) return py_obj @cpython_api([PyObject, rffi.INTP], lltype.Signed, external=False, From fijal at codespeak.net Thu Apr 29 22:54:02 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 22:54:02 +0200 (CEST) Subject: [pypy-svn] r74249 - pypy/branch/cpyext-remove-pyolifeline Message-ID: <20100429205402.163AC282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 22:54:00 2010 New Revision: 74249 Removed: pypy/branch/cpyext-remove-pyolifeline/ Log: Remove merged branch, that was quick From fijal at codespeak.net Thu Apr 29 23:01:42 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Apr 2010 23:01:42 +0200 (CEST) Subject: [pypy-svn] r74250 - in pypy/trunk/pypy: annotation jit/backend/test jit/backend/x86/test rpython/test Message-ID: <20100429210142.DABC6282BEF@codespeak.net> Author: fijal Date: Thu Apr 29 23:01:41 2010 New Revision: 74250 Modified: pypy/trunk/pypy/annotation/annrpython.py pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/rpython/test/test_llinterp.py Log: Attaching entry_point_graph got annoying pretty quickly. Allow build_types to take a flag whether or not this is the main entry point. Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Thu Apr 29 23:01:41 2010 @@ -83,7 +83,8 @@ #___ convenience high-level interface __________________ - def build_types(self, function, input_arg_types, complete_now=True): + def build_types(self, function, input_arg_types, complete_now=True, + main_entry_point=False): """Recursively build annotations about the specific entry point.""" assert isinstance(function, types.FunctionType), "fix that!" @@ -97,6 +98,8 @@ assert isinstance(flowgraph, annmodel.SomeObject) return flowgraph + if main_entry_point: + self.translator.entry_point_graph = flowgraph return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) def get_call_parameters(self, function, args_s, policy): Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Thu Apr 29 23:01:41 2010 @@ -56,9 +56,9 @@ """ % (arglist,)) exec src.compile() in locals() - t.buildannotator().build_types(function, [int] * len(args)) + t.buildannotator().build_types(function, [int] * len(args), + main_entry_point=True) t.buildrtyper(type_system=self.type_system).specialize() - t.entry_point_graph = graphof(t, function) warmrunnerdesc = WarmRunnerDesc(t, translate_support_code=True, CPUClass=self.CPUClass, **kwds) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Thu Apr 29 23:01:41 2010 @@ -82,7 +82,7 @@ for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) - ann.build_types(f, [s_list_of_strings]) + ann.build_types(f, [s_list_of_strings], main_entry_point=True) t.buildrtyper().specialize() if kwds['jit']: apply_jit(t, optimizer=OPTIMIZER_SIMPLE) Modified: pypy/trunk/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_llinterp.py (original) +++ pypy/trunk/pypy/rpython/test/test_llinterp.py Thu Apr 29 23:01:41 2010 @@ -43,7 +43,7 @@ t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) - timelog("annotating", a.build_types, func, argtypes) + timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': viewbefore = getattr(conftest.option, 'view', False) if viewbefore: @@ -62,7 +62,6 @@ t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) - t.entry_point_graph = graph return t, typer, graph _lastinterpreted = [] From afa at codespeak.net Thu Apr 29 23:06:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 29 Apr 2010 23:06:20 +0200 (CEST) Subject: [pypy-svn] r74251 - pypy/trunk/pypy/objspace/std Message-ID: <20100429210620.79424282BEF@codespeak.net> Author: afa Date: Thu Apr 29 23:06:19 2010 New Revision: 74251 Modified: pypy/trunk/pypy/objspace/std/fake.py Log: Fix faking of objects without a __dict__ (file, for example) Modified: pypy/trunk/pypy/objspace/std/fake.py ============================================================================== --- pypy/trunk/pypy/objspace/std/fake.py (original) +++ pypy/trunk/pypy/objspace/std/fake.py Thu Apr 29 23:06:19 2010 @@ -116,7 +116,11 @@ w_self.val = val w_self.space = space def getdict(w_self): - return w_self.space.wrap(w_self.val.__dict__) + try: + d = w_self.val.__dict__ + except AttributeError: + return W_Object.getdict(w_self) + return w_self.space.wrap(d) def unwrap(w_self, space): return w_self.val W_Fake.__name__ = 'W_Fake%s'%(cpy_type.__name__.capitalize()) From fijal at codespeak.net Fri Apr 30 00:32:21 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 00:32:21 +0200 (CEST) Subject: [pypy-svn] r74252 - pypy/trunk/pypy/jit/metainterp Message-ID: <20100429223221.90EF4282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 00:32:18 2010 New Revision: 74252 Modified: pypy/trunk/pypy/jit/metainterp/support.py Log: pass main_entry_point here as well, fixes one test Modified: pypy/trunk/pypy/jit/metainterp/support.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/support.py (original) +++ pypy/trunk/pypy/jit/metainterp/support.py Fri Apr 30 00:32:18 2010 @@ -41,7 +41,7 @@ annpolicy.allow_someobjects = False a = t.buildannotator(policy=annpolicy) argtypes = getargtypes(a, values) - a.build_types(func, argtypes) + a.build_types(func, argtypes, main_entry_point=True) rtyper = t.buildrtyper(type_system = type_system) rtyper.specialize() if inline: From benjamin at codespeak.net Fri Apr 30 00:38:52 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 00:38:52 +0200 (CEST) Subject: [pypy-svn] r74253 - pypy/trunk/pypy/module/cpyext Message-ID: <20100429223852.8902936C200@codespeak.net> Author: benjamin Date: Fri Apr 30 00:38:50 2010 New Revision: 74253 Modified: pypy/trunk/pypy/module/cpyext/import_.py pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/typeobjectdefs.py Log: kill tabs Modified: pypy/trunk/pypy/module/cpyext/import_.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/import_.py (original) +++ pypy/trunk/pypy/module/cpyext/import_.py Fri Apr 30 00:38:50 2010 @@ -30,8 +30,8 @@ else: w_import = space.getattr(w_builtin, space.wrap("__import__")) - # Call the __import__ function with the proper argument list - # Always use absolute import here. + # Call the __import__ function with the proper argument list + # Always use absolute import here. return space.call(w_import, space.newtuple( [w_name, w_globals, w_globals, space.newlist([space.wrap("__doc__")])])) Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri Apr 30 00:38:50 2010 @@ -157,16 +157,16 @@ "x." + NAME + "(y) <==> " + DOC) slotdef_replacements = ( - ("\s+", " "), - ("static [^{]*{", "("), - ("};", ")"), - (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), - (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), - ("'NULL'", "None"), - ("{NULL}", ""), - ("\(wrapperfunc\)", ""), - ("\),", "),\n"), - ) + ("\s+", " "), + ("static [^{]*{", "("), + ("};", ")"), + (r"(?P +..SLOT\([^,]*, )(?P[^,]*), (?P[^,]*), (?P[^,]*)", r"\g'\g', '\g', '\g'"), + (r"(?P *R?[^ ]{3}SLOT(NOTINFIX)?\([^,]*, )(?P[^,]*), (?P[^,]*)", r"\g'\g', '\g'"), + ("'NULL'", "None"), + ("{NULL}", ""), + ("\(wrapperfunc\)", ""), + ("\),", "),\n"), +) """ /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. @@ -177,198 +177,198 @@ """ # Instructions for update: # Copy new slotdefs from typeobject.c -# Remove comments +# Remove comments and tabs # Done. slotdefs_str = """ static slotdef slotdefs[] = { - SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, - "x.__add__(y) <==> x+y"), - SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__mul__(n) <==> x*n"), - SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__rmul__(n) <==> n*x"), - SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, - "x.__getitem__(y) <==> x[y]"), - SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, - "x.__getslice__(i, j) <==> x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, - "x.__setitem__(i, y) <==> x[i]=y"), - SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, - "x.__delitem__(y) <==> del x[y]"), - SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, - wrap_ssizessizeobjargproc, - "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, - "x.__delslice__(i, j) <==> del x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, - "x.__contains__(y) <==> y in x"), - SQSLOT("__iadd__", sq_inplace_concat, NULL, - wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), - SQSLOT("__imul__", sq_inplace_repeat, NULL, - wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), - - MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, - wrap_binaryfunc, - "x.__getitem__(y) <==> x[y]"), - MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_objobjargproc, - "x.__setitem__(i, y) <==> x[i]=y"), - MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_delitem, - "x.__delitem__(y) <==> del x[y]"), - - BINSLOT("__add__", nb_add, slot_nb_add, - "+"), - RBINSLOT("__radd__", nb_add, slot_nb_add, - "+"), - BINSLOT("__sub__", nb_subtract, slot_nb_subtract, - "-"), - RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract, - "-"), - BINSLOT("__mul__", nb_multiply, slot_nb_multiply, - "*"), - RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply, - "*"), - BINSLOT("__div__", nb_divide, slot_nb_divide, - "/"), - RBINSLOT("__rdiv__", nb_divide, slot_nb_divide, - "/"), - BINSLOT("__mod__", nb_remainder, slot_nb_remainder, - "%"), - RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder, - "%"), - BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod, - "divmod(x, y)"), - RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod, - "divmod(y, x)"), - NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc, - "x.__pow__(y[, z]) <==> pow(x, y[, z])"), - NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r, - "y.__rpow__(x[, z]) <==> pow(x, y[, z])"), - UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-x"), - UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+x"), - UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc, - "abs(x)"), - UNSLOT("__nonzero__", nb_nonzero, slot_nb_nonzero, wrap_inquirypred, - "x != 0"), - UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~x"), - BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"), - RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"), - BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"), - RBINSLOT("__rrshift__", nb_rshift, slot_nb_rshift, ">>"), - BINSLOT("__and__", nb_and, slot_nb_and, "&"), - RBINSLOT("__rand__", nb_and, slot_nb_and, "&"), - BINSLOT("__xor__", nb_xor, slot_nb_xor, "^"), - RBINSLOT("__rxor__", nb_xor, slot_nb_xor, "^"), - BINSLOT("__or__", nb_or, slot_nb_or, "|"), - RBINSLOT("__ror__", nb_or, slot_nb_or, "|"), - NBSLOT("__coerce__", nb_coerce, slot_nb_coerce, wrap_coercefunc, - "x.__coerce__(y) <==> coerce(x, y)"), - UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc, - "int(x)"), - UNSLOT("__long__", nb_long, slot_nb_long, wrap_unaryfunc, - "long(x)"), - UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc, - "float(x)"), - UNSLOT("__oct__", nb_oct, slot_nb_oct, wrap_unaryfunc, - "oct(x)"), - UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, - "hex(x)"), - NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, - "x[y:z] <==> x[y.__index__():z.__index__()]"), - IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, - wrap_binaryfunc, "+"), - IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, - wrap_binaryfunc, "-"), - IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply, - wrap_binaryfunc, "*"), - IBSLOT("__idiv__", nb_inplace_divide, slot_nb_inplace_divide, - wrap_binaryfunc, "/"), - IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder, - wrap_binaryfunc, "%"), - IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power, - wrap_binaryfunc, "**"), - IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift, - wrap_binaryfunc, "<<"), - IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift, - wrap_binaryfunc, ">>"), - IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and, - wrap_binaryfunc, "&"), - IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor, - wrap_binaryfunc, "^"), - IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or, - wrap_binaryfunc, "|"), - BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), - RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), - BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), - RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), - IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), - IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), - - TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, - "x.__str__() <==> str(x)"), - TPSLOT("__str__", tp_print, NULL, NULL, ""), - TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, - "x.__repr__() <==> repr(x)"), - TPSLOT("__repr__", tp_print, NULL, NULL, ""), - TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, - "x.__cmp__(y) <==> cmp(x,y)"), - TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, - "x.__hash__() <==> hash(x)"), - FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, - "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), - TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, - wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), - TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), - TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), - TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), - TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, - "x.__setattr__('name', value) <==> x.name = value"), - TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, - "x.__delattr__('name') <==> del x.name"), - TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, - "x.__lt__(y) <==> x x<=y"), - TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, - "x.__eq__(y) <==> x==y"), - TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, - "x.__ne__(y) <==> x!=y"), - TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, - "x.__gt__(y) <==> x>y"), - TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, - "x.__ge__(y) <==> x>=y"), - TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, - "x.__iter__() <==> iter(x)"), - TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, - "x.next() -> the next value, or raise StopIteration"), - TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, - "descr.__get__(obj[, type]) -> value"), - TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, - "descr.__set__(obj, value)"), - TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, - wrap_descr_delete, "descr.__delete__(obj)"), - FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, - "x.__init__(...) initializes x; " - "see x.__class__.__doc__ for signature", - PyWrapperFlag_KEYWORDS), - TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), - TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), - {NULL} + SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, + "x.__add__(y) <==> x+y"), + SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__mul__(n) <==> x*n"), + SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__rmul__(n) <==> n*x"), + SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, + "x.__getitem__(y) <==> x[y]"), + SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, + "x.__getslice__(i, j) <==> x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, + "x.__setitem__(i, y) <==> x[i]=y"), + SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, + "x.__delitem__(y) <==> del x[y]"), + SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, + wrap_ssizessizeobjargproc, + "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, + "x.__delslice__(i, j) <==> del x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, + "x.__contains__(y) <==> y in x"), + SQSLOT("__iadd__", sq_inplace_concat, NULL, + wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), + SQSLOT("__imul__", sq_inplace_repeat, NULL, + wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), + + MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, + wrap_binaryfunc, + "x.__getitem__(y) <==> x[y]"), + MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_objobjargproc, + "x.__setitem__(i, y) <==> x[i]=y"), + MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_delitem, + "x.__delitem__(y) <==> del x[y]"), + + BINSLOT("__add__", nb_add, slot_nb_add, + "+"), + RBINSLOT("__radd__", nb_add, slot_nb_add, + "+"), + BINSLOT("__sub__", nb_subtract, slot_nb_subtract, + "-"), + RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract, + "-"), + BINSLOT("__mul__", nb_multiply, slot_nb_multiply, + "*"), + RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply, + "*"), + BINSLOT("__div__", nb_divide, slot_nb_divide, + "/"), + RBINSLOT("__rdiv__", nb_divide, slot_nb_divide, + "/"), + BINSLOT("__mod__", nb_remainder, slot_nb_remainder, + "%"), + RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder, + "%"), + BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod, + "divmod(x, y)"), + RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod, + "divmod(y, x)"), + NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc, + "x.__pow__(y[, z]) <==> pow(x, y[, z])"), + NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r, + "y.__rpow__(x[, z]) <==> pow(x, y[, z])"), + UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-x"), + UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+x"), + UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc, + "abs(x)"), + UNSLOT("__nonzero__", nb_nonzero, slot_nb_nonzero, wrap_inquirypred, + "x != 0"), + UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~x"), + BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"), + RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"), + BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"), + RBINSLOT("__rrshift__", nb_rshift, slot_nb_rshift, ">>"), + BINSLOT("__and__", nb_and, slot_nb_and, "&"), + RBINSLOT("__rand__", nb_and, slot_nb_and, "&"), + BINSLOT("__xor__", nb_xor, slot_nb_xor, "^"), + RBINSLOT("__rxor__", nb_xor, slot_nb_xor, "^"), + BINSLOT("__or__", nb_or, slot_nb_or, "|"), + RBINSLOT("__ror__", nb_or, slot_nb_or, "|"), + NBSLOT("__coerce__", nb_coerce, slot_nb_coerce, wrap_coercefunc, + "x.__coerce__(y) <==> coerce(x, y)"), + UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc, + "int(x)"), + UNSLOT("__long__", nb_long, slot_nb_long, wrap_unaryfunc, + "long(x)"), + UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc, + "float(x)"), + UNSLOT("__oct__", nb_oct, slot_nb_oct, wrap_unaryfunc, + "oct(x)"), + UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, + "hex(x)"), + NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, + "x[y:z] <==> x[y.__index__():z.__index__()]"), + IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, + wrap_binaryfunc, "+"), + IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, + wrap_binaryfunc, "-"), + IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply, + wrap_binaryfunc, "*"), + IBSLOT("__idiv__", nb_inplace_divide, slot_nb_inplace_divide, + wrap_binaryfunc, "/"), + IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder, + wrap_binaryfunc, "%"), + IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power, + wrap_binaryfunc, "**"), + IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift, + wrap_binaryfunc, "<<"), + IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift, + wrap_binaryfunc, ">>"), + IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and, + wrap_binaryfunc, "&"), + IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor, + wrap_binaryfunc, "^"), + IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or, + wrap_binaryfunc, "|"), + BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), + RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), + BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), + RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), + IBSLOT("__ifloordiv__", nb_inplace_floor_divide, + slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), + IBSLOT("__itruediv__", nb_inplace_true_divide, + slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), + + TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, + "x.__str__() <==> str(x)"), + TPSLOT("__str__", tp_print, NULL, NULL, ""), + TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, + "x.__repr__() <==> repr(x)"), + TPSLOT("__repr__", tp_print, NULL, NULL, ""), + TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, + "x.__cmp__(y) <==> cmp(x,y)"), + TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, + "x.__hash__() <==> hash(x)"), + FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, + "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), + TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, + wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), + TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), + TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), + TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), + TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, + "x.__setattr__('name', value) <==> x.name = value"), + TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, + "x.__delattr__('name') <==> del x.name"), + TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, + "x.__lt__(y) <==> x x<=y"), + TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, + "x.__eq__(y) <==> x==y"), + TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, + "x.__ne__(y) <==> x!=y"), + TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, + "x.__gt__(y) <==> x>y"), + TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, + "x.__ge__(y) <==> x>=y"), + TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, + "x.__iter__() <==> iter(x)"), + TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, + "x.next() -> the next value, or raise StopIteration"), + TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, + "descr.__get__(obj[, type]) -> value"), + TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, + "descr.__set__(obj, value)"), + TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, + wrap_descr_delete, "descr.__delete__(obj)"), + FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, + "x.__init__(...) initializes x; " + "see x.__class__.__doc__ for signature", + PyWrapperFlag_KEYWORDS), + TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), + TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), + {NULL} }; """ for regex, repl in slotdef_replacements: Modified: pypy/trunk/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobjectdefs.py Fri Apr 30 00:38:50 2010 @@ -58,7 +58,7 @@ PyGetSetDef = cpython_struct("PyGetSetDef", ( - ("name", rffi.CCHARP), + ("name", rffi.CCHARP), ("get", getter), ("set", setter), ("doc", rffi.CCHARP), From benjamin at codespeak.net Fri Apr 30 00:41:26 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 00:41:26 +0200 (CEST) Subject: [pypy-svn] r74254 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100429224126.0DD6A282BEF@codespeak.net> Author: benjamin Date: Fri Apr 30 00:41:25 2010 New Revision: 74254 Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py (props changed) pypy/trunk/pypy/module/cpyext/thread.py (props changed) Log: set svn:eol-style From benjamin at codespeak.net Fri Apr 30 00:46:45 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 00:46:45 +0200 (CEST) Subject: [pypy-svn] r74255 - pypy/trunk/pypy/module/thread Message-ID: <20100429224645.27F22282BEF@codespeak.net> Author: benjamin Date: Fri Apr 30 00:46:43 2010 New Revision: 74255 Modified: pypy/trunk/pypy/module/thread/ll_thread.py Log: ident is actually a long Modified: pypy/trunk/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/trunk/pypy/module/thread/ll_thread.py (original) +++ pypy/trunk/pypy/module/thread/ll_thread.py Fri Apr 30 00:46:43 2010 @@ -43,7 +43,7 @@ threadsafe=True) # release the GIL, but most # importantly, reacquire it # around the callback -c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.INT, +c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.LONG, _nowrapper=True) # always call directly TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock', From exarkun at codespeak.net Fri Apr 30 00:55:28 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 30 Apr 2010 00:55:28 +0200 (CEST) Subject: [pypy-svn] r74256 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100429225528.9B63E282BEF@codespeak.net> Author: exarkun Date: Fri Apr 30 00:55:26 2010 New Revision: 74256 Added: pypy/trunk/pypy/module/cpyext/test/banana.c Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: A (skipped) failing test for packages with extension modules in them that do not use their fully qualified name with Py_InitModule Added: pypy/trunk/pypy/module/cpyext/test/banana.c ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/test/banana.c Fri Apr 30 00:55:26 2010 @@ -0,0 +1,10 @@ +#include "Python.h" + +static PyMethodDef banana_functions[] = { + {NULL, NULL} +}; + +void initbanana(void) +{ + Py_InitModule("banana", banana_functions); +} Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 00:55:26 2010 @@ -332,6 +332,17 @@ assert module.__name__ == expected_name + def test_InitModuleInPackage(self): + """ + If `apple.banana` is an extension module which calls Py_InitModule4 with + only "banana" as a name, the resulting module nevertheless is stored at + `sys.modules["apple.banana"]`. + """ + skip("About to implement this") + module = self.import_module(name="apple.banana", filename="banana") + assert module.__name__ == "apple.banana" + + def test_modinit_func(self): """ A module can use the PyMODINIT_FUNC macro to declare or define its From benjamin at codespeak.net Fri Apr 30 00:59:43 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 00:59:43 +0200 (CEST) Subject: [pypy-svn] r74257 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100429225943.E760F282BEF@codespeak.net> Author: benjamin Date: Fri Apr 30 00:59:42 2010 New Revision: 74257 Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py Log: xfail tests Modified: pypy/trunk/pypy/module/cpyext/test/test_thread.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_thread.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_thread.py Fri Apr 30 00:59:42 2010 @@ -1,5 +1,7 @@ +import py -import thread, threading +import thread +import threading from pypy.module.thread.ll_thread import allocate_ll_lock from pypy.module.cpyext.test.test_api import BaseApiTest @@ -22,15 +24,15 @@ assert results[0][0] != results[1][0] - - # def test_acquire_lock(self, space, api): - # lock = allocate_ll_lock() - # assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 - # assert api.PyThread_acquire_lock(lock, space.w_int(1)) == 0 - - - # def test_release_lock(self, space, api): - # lock = allocate_ll_lock() - # api.PyThread_acquire_lock(lock, space.w_int(0)) - # api.PyThread_release_lock(lock) - # assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 + @py.test.mark.xfail + def test_acquire_lock(self, space, api): + lock = allocate_ll_lock() + assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 + assert api.PyThread_acquire_lock(lock, space.w_int(1)) == 0 + + @py.test.mark.xfail + def test_release_lock(self, space, api): + lock = allocate_ll_lock() + api.PyThread_acquire_lock(lock, space.w_int(0)) + api.PyThread_release_lock(lock) + assert api.PyThread_acquire_lock(lock, space.w_int(0)) == 1 From benjamin at codespeak.net Fri Apr 30 01:00:21 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 01:00:21 +0200 (CEST) Subject: [pypy-svn] r74258 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100429230021.E9603282BEF@codespeak.net> Author: benjamin Date: Fri Apr 30 01:00:20 2010 New Revision: 74258 Modified: pypy/trunk/pypy/module/cpyext/test/banana.c (props changed) Log: set native eol From fijal at codespeak.net Fri Apr 30 01:02:54 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 01:02:54 +0200 (CEST) Subject: [pypy-svn] r74259 - pypy/trunk/pypy/translator Message-ID: <20100429230254.021A6282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 01:02:53 2010 New Revision: 74259 Modified: pypy/trunk/pypy/translator/driver.py Log: Ugh. Fix gc failures by not passing stuff that we don't know. TDD fail I believe Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Fri Apr 30 01:02:53 2010 @@ -551,7 +551,7 @@ """ cbuilder = self.cbuilder kwds = {} - if self.standalone: + if self.standalone and self.exe_name is not None: kwds['exe_name'] = self.compute_exe_name().basename cbuilder.compile(**kwds) From fijal at codespeak.net Fri Apr 30 01:10:19 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 01:10:19 +0200 (CEST) Subject: [pypy-svn] r74260 - pypy/trunk/pypy/objspace/std Message-ID: <20100429231019.1F62C282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 01:10:17 2010 New Revision: 74260 Modified: pypy/trunk/pypy/objspace/std/objspace.py Log: Write down some docstrings Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Fri Apr 30 01:10:17 2010 @@ -453,21 +453,33 @@ raiseattrerror(self, w_obj, name) def finditem_str(self, w_obj, key): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Perform a getitem on w_obj with key (string). Returns found + element or None on element not found. + + performance shortcut to avoid creating the OperationError(KeyError) + and allocating W_StringObject + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): return w_obj.getitem_str(key) return ObjSpace.finditem_str(self, w_obj, key) def finditem(self, w_obj, w_key): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Perform a getitem on w_obj with w_key (any object). Returns found + element or None on element not found. + + performance shortcut to avoid creating the OperationError(KeyError). + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) def setitem_str(self, w_obj, key, w_value, shadows_type=True): - # performance shortcut to avoid creating the OperationError(KeyError) + """ Same as setitem, but takes string instead of any wrapped object + + XXX what shadows_type means??? + """ if (isinstance(w_obj, W_DictMultiObject) and not w_obj.user_overridden_class): w_obj.setitem_str(key, w_value, shadows_type) From afa at codespeak.net Fri Apr 30 01:13:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 01:13:49 +0200 (CEST) Subject: [pypy-svn] r74261 - in pypy/trunk/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20100429231349.8D061282BEF@codespeak.net> Author: afa Date: Fri Apr 30 01:13:47 2010 New Revision: 74261 Modified: pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/rlib/test/test_rbigint.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py Log: Fix one test in test_rbigint: "int_real" does not implement arithmetic and should be skipped. Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Fri Apr 30 01:13:47 2010 @@ -380,6 +380,8 @@ r_int = build_int('r_int', True, LONG_BIT) r_uint = build_int('r_uint', False, LONG_BIT) +r_int_real = build_int('r_int_real', r_int.SIGN, r_int.BITS, + force_creation=True) r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) Modified: pypy/trunk/pypy/rlib/test/test_rbigint.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rbigint.py (original) +++ pypy/trunk/pypy/rlib/test/test_rbigint.py Fri Apr 30 01:13:47 2010 @@ -473,11 +473,11 @@ def test_args_from_rarith_int(self): from pypy.rpython.tool.rfficache import platform - from pypy.rlib.rarithmetic import r_int + from pypy.rlib.rarithmetic import r_int, r_int_real classlist = platform.numbertype_to_rclass.values() fnlist = [] for r in classlist: - if r is r_int: # and also r_longlong on 64-bit + if r in (r_int, r_int_real): # and also r_longlong on 64-bit continue if r is int: mask = sys.maxint*2+1 Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Fri Apr 30 01:13:47 2010 @@ -386,7 +386,7 @@ NUMBER_TYPES = setup() platform.numbertype_to_rclass[lltype.Signed] = int # avoid "r_long" for common cases -r_int_real = rarithmetic.build_int("r_int_real", r_int.SIGN, r_int.BITS, True) +r_int_real = rarithmetic.r_int_real INT_real = lltype.build_number("INT", r_int_real) platform.numbertype_to_rclass[INT_real] = r_int_real NUMBER_TYPES.append(INT_real) From benjamin at codespeak.net Fri Apr 30 01:27:08 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 30 Apr 2010 01:27:08 +0200 (CEST) Subject: [pypy-svn] r74262 - in pypy/trunk/pypy/module/thread: . test Message-ID: <20100429232708.4AD73282BEF@codespeak.net> Author: benjamin Date: Fri Apr 30 01:27:06 2010 New Revision: 74262 Modified: pypy/trunk/pypy/module/thread/ll_thread.py pypy/trunk/pypy/module/thread/test/test_ll_thread.py Log: always deal with ident as a long Modified: pypy/trunk/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/trunk/pypy/module/thread/ll_thread.py (original) +++ pypy/trunk/pypy/module/thread/ll_thread.py Fri Apr 30 01:27:06 2010 @@ -35,10 +35,10 @@ ident = thread.start_new_thread(func, ()) except thread.error: ident = -1 - return rffi.cast(rffi.INT, ident) + return rffi.cast(rffi.LONG, ident) CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void)) -c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.INT, +c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG, _callable=_emulated_start_new_thread, threadsafe=True) # release the GIL, but most # importantly, reacquire it Modified: pypy/trunk/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/trunk/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/trunk/pypy/module/thread/test/test_ll_thread.py Fri Apr 30 01:27:06 2010 @@ -47,15 +47,16 @@ state.freed_counter += 1 def bootstrap(): - state.my_thread_ident = get_ident() + state.my_thread_ident = state.z.ident = get_ident() assert state.my_thread_ident == get_ident() + assert get_ident() == state.z.ident state.seen_value = state.z.value state.z = None state.done = 1 def g(i): state.z = Z(i) - start_new_thread(bootstrap, ()) + return start_new_thread(bootstrap, ()) g._dont_inline_ = True def f(): @@ -65,7 +66,7 @@ for i in range(50): state.done = 0 state.seen_value = 0 - g(i) + ident = g(i) gc.collect() willing_to_wait_more = 1000 while not state.done: @@ -74,6 +75,7 @@ raise Exception("thread didn't start?") time.sleep(0.01) assert state.my_thread_ident != main_ident + assert state.my_thread_ident == ident assert state.seen_value == i # try to force Boehm to do some freeing for i in range(3): From fijal at codespeak.net Fri Apr 30 04:33:49 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 04:33:49 +0200 (CEST) Subject: [pypy-svn] r74263 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100430023349.24B00282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 04:33:47 2010 New Revision: 74263 Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Grumble. Fix pyolifeline, this code is untested! Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 04:33:47 2010 @@ -95,7 +95,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = w_obj.get_pyolifeline() + lifeline = api.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: From fijal at codespeak.net Fri Apr 30 04:54:24 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 04:54:24 +0200 (CEST) Subject: [pypy-svn] r74264 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100430025424.380E2282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 04:54:22 2010 New Revision: 74264 Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: A function that takes self is a method. A nice way to share methods is via inheritance. Modified: pypy/trunk/pypy/module/cpyext/test/test_api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_api.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_api.py Fri Apr 30 04:54:22 2010 @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.state import State from pypy.module.cpyext import api -from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, check_and_print_leaks +from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest PyObject = api.PyObject from pypy.interpreter.error import OperationError from pypy.module.cpyext.state import State @@ -15,7 +15,7 @@ def PyPy_GetReference(space, arg): assert lltype.typeOf(arg) == PyObject -class BaseApiTest: +class BaseApiTest(LeakCheckingTest): def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") @@ -47,7 +47,7 @@ except OperationError, e: print e.errorstr(self.space) raise - if check_and_print_leaks(self): + if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." class TestConversion(BaseApiTest): Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 04:54:22 2010 @@ -78,56 +78,57 @@ self.frozen_lltallocations = lltype.ALLOCATED.copy() lltype.TRACK_ALLOCATIONS = True -def check_and_print_leaks(self): - # check for sane refcnts - leaking = False - state = self.space.fromcache(State) - import gc - gc.collect() - lost_objects_w = identity_dict() - lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - for w_obj, obj in state.py_objects_w2r.iteritems(): - base_refcnt = self.frozen_refcounts.get(w_obj) - delta = obj.c_ob_refcnt - if base_refcnt is not None: - delta -= base_refcnt - lost_objects_w.pop(w_obj) - if delta != 0: +class LeakCheckingTest(object): + def check_and_print_leaks(self): + # check for sane refcnts + import gc + + leaking = False + state = self.space.fromcache(State) + gc.collect() + lost_objects_w = identity_dict() + lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) + for w_obj, obj in state.py_objects_w2r.iteritems(): + base_refcnt = self.frozen_refcounts.get(w_obj) + delta = obj.c_ob_refcnt + if base_refcnt is not None: + delta -= base_refcnt + lost_objects_w.pop(w_obj) + if delta != 0: + leaking = True + print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) + lifeline = api.lifeline_dict.get(w_obj) + if lifeline is not None: + refcnt = lifeline.pyo.c_ob_refcnt + if refcnt > 0: + print >>sys.stderr, "\tThe object also held by C code." + else: + referrers_repr = [] + for o in gc.get_referrers(w_obj): + try: + repr_str = repr(o) + except TypeError, e: + repr_str = "%s (type of o is %s)" % (str(e), type(o)) + referrers_repr.append(repr_str) + referrers = ", ".join(referrers_repr) + print >>sys.stderr, "\tThe object is referenced by these objects:", \ + referrers + for w_obj in lost_objects_w: + print >>sys.stderr, "Lost object %r" % (w_obj, ) leaking = True - print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = api.lifeline_dict.get(w_obj) - if lifeline is not None: - refcnt = lifeline.pyo.c_ob_refcnt - if refcnt > 0: - print >>sys.stderr, "\tThe object also held by C code." - else: - referrers_repr = [] - for o in gc.get_referrers(w_obj): - try: - repr_str = repr(o) - except TypeError, e: - repr_str = "%s (type of o is %s)" % (str(e), type(o)) - referrers_repr.append(repr_str) - referrers = ", ".join(referrers_repr) - print >>sys.stderr, "\tThe object is referenced by these objects:", \ - referrers - for w_obj in lost_objects_w: - print >>sys.stderr, "Lost object %r" % (w_obj, ) - leaking = True - for llvalue in set(ll2ctypes.ALLOCATED.values()) - self.frozen_ll2callocations: - if getattr(llvalue, "_traceback", None): # this means that the allocation should be tracked + for llvalue in set(ll2ctypes.ALLOCATED.values()) - self.frozen_ll2callocations: + if getattr(llvalue, "_traceback", None): # this means that the allocation should be tracked + leaking = True + print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) + print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) + for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): leaking = True - print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) + print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): - leaking = True - print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) - print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - - return leaking + return leaking -class AppTestCpythonExtensionBase: +class AppTestCpythonExtensionBase(LeakCheckingTest): def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") @@ -234,7 +235,7 @@ pass except AttributeError: pass - if check_and_print_leaks(self): + if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." From exarkun at codespeak.net Fri Apr 30 05:19:05 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 30 Apr 2010 05:19:05 +0200 (CEST) Subject: [pypy-svn] r74265 - pypy/branch/py-packagecontext Message-ID: <20100430031905.70F59282BEF@codespeak.net> Author: exarkun Date: Fri Apr 30 05:19:04 2010 New Revision: 74265 Added: pypy/branch/py-packagecontext/ (props changed) - copied from r74264, pypy/trunk/ Log: Branching to 'py-packagecontext' From exarkun at codespeak.net Fri Apr 30 05:21:13 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 30 Apr 2010 05:21:13 +0200 (CEST) Subject: [pypy-svn] r74266 - in pypy/branch/py-packagecontext/pypy/module/cpyext: . test Message-ID: <20100430032113.8FBBB282BEF@codespeak.net> Author: exarkun Date: Fri Apr 30 05:21:12 2010 New Revision: 74266 Added: pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/api.py pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py pypy/branch/py-packagecontext/pypy/module/cpyext/state.py pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Log: Add a test for recursive extension module imports; and probably fix both of these new tests Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/api.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/api.py Fri Apr 30 05:21:12 2010 @@ -745,24 +745,28 @@ @unwrap_spec(ObjSpace, str, str) def load_extension_module(space, path, name): state = space.fromcache(State) - from pypy.rlib import libffi + state.package_context = name try: - dll = libffi.CDLL(path, False) - except libffi.DLOpenError, e: - raise operationerrfmt( - space.w_ImportError, - "unable to load extension module '%s': %s", - path, e.msg) - try: - initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) - except KeyError: - raise operationerrfmt( - space.w_ImportError, - "function init%s not found in library %s", - name, path) - initfunc = rffi.cast(initfunctype, initptr) - generic_cpy_call(space, initfunc) - state.check_and_raise_exception() + from pypy.rlib import libffi + try: + dll = libffi.CDLL(path, False) + except libffi.DLOpenError, e: + raise operationerrfmt( + space.w_ImportError, + "unable to load extension module '%s': %s", + path, e.msg) + try: + initptr = libffi.dlsym(dll.lib, 'init%s' % (name.split('.')[-1],)) + except KeyError: + raise operationerrfmt( + space.w_ImportError, + "function init%s not found in library %s", + name, path) + initfunc = rffi.cast(initfunctype, initptr) + generic_cpy_call(space, initfunc) + state.check_and_raise_exception() + finally: + state.package_context = None @specialize.ll() def generic_cpy_call(space, func, *args): Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/modsupport.py Fri Apr 30 05:21:12 2010 @@ -5,14 +5,18 @@ from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyCFunction from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError def PyImport_AddModule(space, name): w_name = space.wrap(name) - w_mod = space.wrap(Module(space, w_name)) - w_modules = space.sys.get('modules') - space.setitem(w_modules, w_name, w_mod) + + w_mod = space.finditem_str(w_modules, name) + if w_mod is None: + w_mod = space.wrap(Module(space, w_name)) + space.setitem(w_modules, w_name, w_mod) + return w_mod @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, @@ -20,7 +24,9 @@ def Py_InitModule4(space, name, methods, doc, w_self, apiver): from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr modname = rffi.charp2str(name) - w_mod = PyImport_AddModule(space, modname) + state = space.fromcache(State) + w_mod = PyImport_AddModule(space, state.package_context) + dict_w = {} convert_method_defs(space, dict_w, methods, lltype.nullptr(PyTypeObjectPtr.TO), w_self) for key, w_value in dict_w.items(): Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/state.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/state.py Fri Apr 30 05:21:12 2010 @@ -22,6 +22,17 @@ self.exc_value = None self.new_method_def = lltype.nullptr(PyMethodDef) + # When importing a package, use this to keep track of its name. This is + # necessary because an extension module in a package might not supply + # its own fully qualified name to Py_InitModule. If it doesn't, we need + # to be able to figure out what module is being initialized. When a + # package import is in progress, this is set to the name of the package. + # The rest of the time, it's None. Packages may be imported + # recursively, in which case the outer state is preserved somewhere in + # the stack and then restored when the inner import is complete. + self.package_context = None + + def _freeze_(self): assert not self.borrowed_objects and not self.borrow_mapping self.py_objects_r2w.clear() # is not valid anymore after translation Added: pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c ============================================================================== --- (empty file) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c Fri Apr 30 05:21:12 2010 @@ -0,0 +1,11 @@ +#include "Python.h" + +static PyMethodDef date_functions[] = { + {NULL, NULL} +}; + +void initdate(void) +{ + Py_InitModule("date", date_functions); + PyImport_ImportModule("apple.banana"); +} Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 05:21:12 2010 @@ -10,7 +10,7 @@ from pypy.translator import platform from pypy.translator.gensupp import uniquemodulename from pypy.tool.udir import udir -from pypy.module.cpyext import api +from pypy.module.cpyext import api, typeobject from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from pypy.translator.goal import autopath @@ -50,8 +50,9 @@ raises(ImportError, cpyext.load_module, self.libc, "invalid.function") def compile_module(modname, **kwds): + modname = modname.split('.')[-1] eci = ExternalCompilationInfo( - export_symbols=['init%s' % (modname.split('.')[-1],)], + export_symbols=['init%s' % (modname,)], include_dirs=api.include_dirs, **kwds ) @@ -97,7 +98,7 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = api.lifeline_dict.get(w_obj) + lifeline = typeobject.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -133,6 +134,19 @@ cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") + def compile_module(self, name, **kwds): + state = self.space.fromcache(State) + api_library = state.api_lib + if sys.platform == 'win32': + kwds["libraries"] = [api_library] + # '%s' undefined; assuming extern returning int + kwds["compile_extra"] = ["/we4013"] + else: + kwds["link_files"] = [str(api_library + '.so')] + kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] + return compile_module(name, **kwds) + + def import_module(self, name, init=None, body='', load_it=True, filename=None): """ init specifies the overall template of the module. @@ -160,20 +174,11 @@ / 'cpyext'/ 'test' / (filename + ".c") kwds = dict(separate_module_files=[filename]) - state = self.space.fromcache(State) - api_library = state.api_lib - if sys.platform == 'win32': - kwds["libraries"] = [api_library] - # '%s' undefined; assuming extern returning int - kwds["compile_extra"] = ["/we4013"] - else: - kwds["link_files"] = [str(api_library + '.so')] - kwds["compile_extra"] = ["-Werror=implicit-function-declaration"] - mod = compile_module(name, **kwds) + mod = self.compile_module(name, **kwds) - self.name = name if load_it: api.load_extension_module(self.space, mod, name) + self.imported_module_names.append(name) return self.space.getitem( self.space.sys.get('modules'), self.space.wrap(name)) @@ -204,9 +209,21 @@ init = """Py_InitModule("%s", methods);""" % (modname,) return self.import_module(name=modname, init=init, body=body) + def record_imported_module(self, name): + self.imported_module_names.append(name) + def setup_method(self, func): + # A list of modules which the test caused to be imported (in + # self.space). These will be cleaned up automatically in teardown. + self.imported_module_names = [] self.w_import_module = self.space.wrap(self.import_module) self.w_import_extension = self.space.wrap(self.import_extension) + self.w_compile_module = self.space.wrap(self.compile_module) + self.w_record_imported_module = self.space.wrap( + self.record_imported_module) + self.w_here = self.space.wrap( + str(py.path.local(autopath.pypydir)) + '/module/cpyext/test/') + # create the file lock before we count allocations self.space.call_method(self.space.sys.get("stdout"), "flush") @@ -214,13 +231,17 @@ freeze_refcnts(self) #self.check_and_print_leaks() + def unimport_module(self, name): + w_modules = self.space.sys.get('modules') + w_name = self.space.wrap(name) + w_mod = self.space.getitem(w_modules, w_name) + self.space.delitem(w_modules, w_name) + Py_DecRef(self.space, w_mod) + def teardown_method(self, func): try: - w_mod = self.space.getitem(self.space.sys.get('modules'), - self.space.wrap(self.name)) - self.space.delitem(self.space.sys.get('modules'), - self.space.wrap(self.name)) - Py_DecRef(self.space, w_mod) + for name in self.imported_module_names: + self.unimport_module(name) state = self.space.fromcache(State) for w_obj in state.non_heaptypes: Py_DecRef(self.space, w_obj) @@ -339,11 +360,39 @@ only "banana" as a name, the resulting module nevertheless is stored at `sys.modules["apple.banana"]`. """ - skip("About to implement this") module = self.import_module(name="apple.banana", filename="banana") assert module.__name__ == "apple.banana" + def test_recursivePackageImport(self): + """ + If `cherry.date` is an extension module which imports `apple.banana`, + the latter is added to `sys.modules` for the `"apple.banana"` key. + """ + # Build the extensions. + banana = self.compile_module( + "apple.banana", separate_module_files=[self.here + 'banana.c']) + self.record_imported_module("apple.banana") + date = self.compile_module( + "cherry.date", separate_module_files=[self.here + 'date.c']) + self.record_imported_module("cherry.date") + + # Set up some package state so that the extensions can actually be + # imported. + import sys, types, os + cherry = sys.modules['cherry'] = types.ModuleType('cherry') + cherry.__path__ = [os.path.dirname(date)] + + apple = sys.modules['apple'] = types.ModuleType('apple') + apple.__path__ = [os.path.dirname(banana)] + + import cherry.date + import apple.banana + + assert sys.modules['apple.banana'].__name__ == 'apple.banana' + assert sys.modules['cherry.date'].__name__ == 'cherry.date' + + def test_modinit_func(self): """ A module can use the PyMODINIT_FUNC macro to declare or define its From exarkun at codespeak.net Fri Apr 30 05:26:55 2010 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 30 Apr 2010 05:26:55 +0200 (CEST) Subject: [pypy-svn] r74267 - pypy/branch/py-packagecontext/pypy/module/cpyext/test Message-ID: <20100430032655.00E35282BEF@codespeak.net> Author: exarkun Date: Fri Apr 30 05:26:54 2010 New Revision: 74267 Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Log: underscore test names Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 05:26:54 2010 @@ -343,7 +343,7 @@ assert module.return_cookie() == 3.14 - def test_InitModule4Dotted(self): + def test_InitModule4_dotted(self): """ If the module name passed to Py_InitModule4 includes a package, only the module name (the part after the last dot) is considered when @@ -354,7 +354,7 @@ assert module.__name__ == expected_name - def test_InitModuleInPackage(self): + def test_InitModule4_in_package(self): """ If `apple.banana` is an extension module which calls Py_InitModule4 with only "banana" as a name, the resulting module nevertheless is stored at @@ -364,7 +364,7 @@ assert module.__name__ == "apple.banana" - def test_recursivePackageImport(self): + def test_recursive_package_import(self): """ If `cherry.date` is an extension module which imports `apple.banana`, the latter is added to `sys.modules` for the `"apple.banana"` key. From fijal at codespeak.net Fri Apr 30 05:45:51 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 05:45:51 +0200 (CEST) Subject: [pypy-svn] r74268 - pypy/trunk/pypy/translator/c/gcc/test Message-ID: <20100430034551.A29BF282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 05:45:50 2010 New Revision: 74268 Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Log: I'm not overly happy with the solution, but cleanup secondary_entrypoints for this test. Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Fri Apr 30 05:45:50 2010 @@ -38,14 +38,15 @@ config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() - for f, inputtypes in cls.secondary_entrypoints: + sec_ep = getattr(cls, 'secondary_entrypoints', []) + for f, inputtypes in sec_ep: a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config, - secondary_entrypoints=cls.secondary_entrypoints) + secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) @@ -165,7 +166,13 @@ res = self.run('callback_simple') assert res == 4900 - def define_secondary_entrypoint_callback(self): + def define_secondary_entrypoint_callback(cls): + # XXX this is baaaad, cleanup global state + try: + del secondary_entrypoints["x42"] + except KeyError: + pass + @entrypoint("x42", [lltype.Signed, lltype.Signed], c_name='callback') def mycallback(a, b): llop.gc_stack_bottom(lltype.Void) @@ -186,7 +193,7 @@ compilation_info=eci) S = lltype.GcStruct('S', ('x', lltype.Signed)) - self.secondary_entrypoints.extend(secondary_entrypoints["x42"]) + cls.secondary_entrypoints = secondary_entrypoints["x42"] def f(): p = lltype.malloc(S) From fijal at codespeak.net Fri Apr 30 05:49:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 05:49:23 +0200 (CEST) Subject: [pypy-svn] r74269 - pypy/trunk/pypy/translator Message-ID: <20100430034923.065BB282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 05:49:22 2010 New Revision: 74269 Modified: pypy/trunk/pypy/translator/driver.py Log: fix tests Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Fri Apr 30 05:49:22 2010 @@ -336,6 +336,7 @@ "int (and not, e.g., None or always raise an " "exception).") annotator.simplify() + return s # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") From fijal at codespeak.net Fri Apr 30 05:58:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 05:58:30 +0200 (CEST) Subject: [pypy-svn] r74270 - pypy/branch/py-packagecontext/pypy/module/cpyext/test Message-ID: <20100430035830.38CD8282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 05:58:28 2010 New Revision: 74270 Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_import.py Log: Skip tests that are suspects of leakage (just because) Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 05:58:28 2010 @@ -365,6 +365,7 @@ def test_recursive_package_import(self): + skip("not yet") """ If `cherry.date` is an extension module which imports `apple.banana`, the latter is added to `sys.modules` for the `"apple.banana"` key. Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_import.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_import.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_import.py Fri Apr 30 05:58:28 2010 @@ -14,6 +14,7 @@ class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): + skip("leak?") path = self.import_module(name='test_import_module', load_it=False) import sys sys.path.append(path) From fijal at codespeak.net Fri Apr 30 06:00:31 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Apr 2010 06:00:31 +0200 (CEST) Subject: [pypy-svn] r74271 - pypy/branch/py-packagecontext/pypy/module/cpyext/test Message-ID: <20100430040031.24F50282BEF@codespeak.net> Author: fijal Date: Fri Apr 30 06:00:29 2010 New Revision: 74271 Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Log: remove blank try: except:, they don't seem to be needed any more Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 06:00:29 2010 @@ -239,23 +239,18 @@ Py_DecRef(self.space, w_mod) def teardown_method(self, func): - try: - for name in self.imported_module_names: - self.unimport_module(name) - state = self.space.fromcache(State) - for w_obj in state.non_heaptypes: - Py_DecRef(self.space, w_obj) - state.non_heaptypes[:] = [] - while state.borrowed_objects: - addr = state.borrowed_objects.keys()[0] - w_obj = state.py_objects_r2w[addr] - Py_DecRef(self.space, w_obj) - state.borrowed_objects = {} - state.borrow_mapping = {} - except OperationError: - pass - except AttributeError: - pass + for name in self.imported_module_names: + self.unimport_module(name) + state = self.space.fromcache(State) + for w_obj in state.non_heaptypes: + Py_DecRef(self.space, w_obj) + state.non_heaptypes[:] = [] + while state.borrowed_objects: + addr = state.borrowed_objects.keys()[0] + w_obj = state.py_objects_r2w[addr] + Py_DecRef(self.space, w_obj) + state.borrowed_objects = {} + state.borrow_mapping = {} if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." From arigo at codespeak.net Fri Apr 30 10:37:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 10:37:41 +0200 (CEST) Subject: [pypy-svn] r74272 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp metainterp/test Message-ID: <20100430083741.1641F282BF1@codespeak.net> Author: arigo Date: Fri Apr 30 10:37:37 2010 New Revision: 74272 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Log: Kill the int_rename operation, and do its job with an explicit list of operations int_push/int_copy/int_copy/.../int_copy/int_pop. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Fri Apr 30 10:37:37 2010 @@ -262,16 +262,18 @@ for kind in KINDS: if kind in renamings: frm, to = renamings[kind] - # If there is no cycle among the renamings, produce a series - # of %s_copy. Otherwise, just one %s_rename. + # Produce a series of %s_copy. If there is a cycle, it + # is handled with a %s_push to save the first value of + # the cycle, some number of %s_copy, and finally a + # %s_pop to load the last value. result = reorder_renaming_list(frm, to) - if result is not None: - for v, w in result: + for v, w in result: + if w is None: + self.emitline('%s_push' % kind, v) + elif v is None: + self.emitline('%s_pop' % kind, w) + else: self.emitline('%s_copy' % kind, v, w) - else: - frm = ListOfKind(kind, frm) - to = ListOfKind(kind, to) - self.emitline('%s_rename' % kind, frm, to) def emitline(self, *line): self.ssarepr.insns.append(line) @@ -326,6 +328,10 @@ else: still_pending_indices.append(i) if len(pending_indices) == len(still_pending_indices): - return None # no progress -- there is a cycle + # no progress -- there is a cycle + assert None not in not_read + result.append((frm[pending_indices[0]], None)) + frm[pending_indices[0]] = None + continue pending_indices = still_pending_indices return result Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri Apr 30 10:37:37 2010 @@ -56,9 +56,12 @@ result = reorder_renaming_list([4, 5, 1, 2], [1, 2, 3, 4]) assert result == [(1, 3), (4, 1), (2, 4), (5, 2)] result = reorder_renaming_list([1, 2], [2, 1]) - assert result == None - result = reorder_renaming_list([4, 3, 1, 2, 6], [1, 2, 3, 4, 5]) - assert result == None + assert result == [(1, None), (2, 1), (None, 2)] + result = reorder_renaming_list([4, 3, 6, 1, 2, 5, 7], + [1, 2, 5, 3, 4, 6, 8]) + assert result == [(7, 8), + (4, None), (2, 4), (3, 2), (1, 3), (None, 1), + (6, None), (5, 6), (None, 5)] def test_repr(): assert repr(Register('int', 13)) == '%i13' Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_regalloc.py Fri Apr 30 10:37:37 2010 @@ -77,7 +77,9 @@ L1: int_gt %i0, $0, %i2 goto_if_not L2, %i2 - int_rename I[%i1, %i0], I[%i0, %i1] + int_push %i1 + int_copy %i0, %i1 + int_pop %i0 goto L1 L2: int_return %i1 @@ -110,7 +112,10 @@ L1: int_gt %i0, $0, %i3 goto_if_not L2, %i3 - int_rename I[%i1, %i2, %i0], I[%i0, %i1, %i2] + int_push %i1 + int_copy %i2, %i1 + int_copy %i0, %i2 + int_pop %i0 goto L1 L2: int_return %i1 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri Apr 30 10:37:37 2010 @@ -266,13 +266,13 @@ position = self.handle_exception_in_frame(e, code) def get_result_i(self): - return self.registers_i[0] + return self.tmpreg_i def get_result_r(self): - return self.registers_r[0] + return self.tmpreg_r def get_result_f(self): - return self.registers_f[0] + return self.tmpreg_f def _get_result_anytype(self): "NOT_RPYTHON" @@ -291,6 +291,7 @@ while i >= 0: self.registers_r[i] = NULL i -= 1 + self.tmpreg_r = NULL self.exception_last_value = None def handle_exception_in_frame(self, e, code): @@ -466,6 +467,26 @@ bhimpl_ref_guard_value = bhimpl_ref_copy bhimpl_float_guard_value = bhimpl_float_copy + @arguments("self", "i") + def bhimpl_int_push(self, a): + self.tmpreg_i = a + @arguments("self", "r") + def bhimpl_ref_push(self, a): + self.tmpreg_r = a + @arguments("self", "f") + def bhimpl_float_push(self, a): + self.tmpreg_f = a + + @arguments("self", returns="i") + def bhimpl_int_pop(self): + return self.tmpreg_i + @arguments("self", returns="r") + def bhimpl_ref_pop(self): + return self.tmpreg_r + @arguments("self", returns="f") + def bhimpl_float_pop(self): + return self.tmpreg_f + # ---------- # float operations @@ -526,21 +547,21 @@ @arguments("self", "i") def bhimpl_int_return(self, a): - self.registers_i[0] = a + self.tmpreg_i = a if not we_are_translated(): self._return_type = "int" raise LeaveFrame @arguments("self", "r") def bhimpl_ref_return(self, a): - self.registers_r[0] = a + self.tmpreg_r = a if not we_are_translated(): self._return_type = "ref" raise LeaveFrame @arguments("self", "f") def bhimpl_float_return(self, a): - self.registers_f[0] = a + self.tmpreg_f = a if not we_are_translated(): self._return_type = "float" raise LeaveFrame Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/test/test_basic.py Fri Apr 30 10:37:37 2010 @@ -1389,6 +1389,16 @@ # this checks that the logic triggered by make_a_counter_per_value() # works and prevents generating tons of bridges + def test_swap_values(self): + def f(x, y): + if x > 5: + x, y = y, x + return x - y + res = self.interp_operations(f, [10, 2]) + assert res == -8 + res = self.interp_operations(f, [3, 2]) + assert res == 1 + class TestOOtype(BasicTests, OOJitMixin): From afa at codespeak.net Fri Apr 30 10:37:48 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 10:37:48 +0200 (CEST) Subject: [pypy-svn] r74273 - in pypy/trunk/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20100430083748.56B23282BF5@codespeak.net> Author: afa Date: Fri Apr 30 10:37:46 2010 New Revision: 74273 Modified: pypy/trunk/pypy/rlib/rarithmetic.py pypy/trunk/pypy/rlib/test/test_rbigint.py pypy/trunk/pypy/rpython/lltypesystem/rffi.py Log: revert r74261, and fix the test differently Modified: pypy/trunk/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/trunk/pypy/rlib/rarithmetic.py (original) +++ pypy/trunk/pypy/rlib/rarithmetic.py Fri Apr 30 10:37:46 2010 @@ -380,8 +380,6 @@ r_int = build_int('r_int', True, LONG_BIT) r_uint = build_int('r_uint', False, LONG_BIT) -r_int_real = build_int('r_int_real', r_int.SIGN, r_int.BITS, - force_creation=True) r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) Modified: pypy/trunk/pypy/rlib/test/test_rbigint.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_rbigint.py (original) +++ pypy/trunk/pypy/rlib/test/test_rbigint.py Fri Apr 30 10:37:46 2010 @@ -473,7 +473,8 @@ def test_args_from_rarith_int(self): from pypy.rpython.tool.rfficache import platform - from pypy.rlib.rarithmetic import r_int, r_int_real + from pypy.rlib.rarithmetic import r_int + from pypy.rpython.lltypesystem.rffi import r_int_real classlist = platform.numbertype_to_rclass.values() fnlist = [] for r in classlist: Modified: pypy/trunk/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rffi.py Fri Apr 30 10:37:46 2010 @@ -386,7 +386,7 @@ NUMBER_TYPES = setup() platform.numbertype_to_rclass[lltype.Signed] = int # avoid "r_long" for common cases -r_int_real = rarithmetic.r_int_real +r_int_real = rarithmetic.build_int("r_int_real", r_int.SIGN, r_int.BITS, True) INT_real = lltype.build_number("INT", r_int_real) platform.numbertype_to_rclass[INT_real] = r_int_real NUMBER_TYPES.append(INT_real) From jandem at codespeak.net Fri Apr 30 10:43:37 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 10:43:37 +0200 (CEST) Subject: [pypy-svn] r74274 - pypy/trunk/pypy/module/cpyext Message-ID: <20100430084337.CEE92282BF1@codespeak.net> Author: jandem Date: Fri Apr 30 10:43:29 2010 New Revision: 74274 Modified: pypy/trunk/pypy/module/cpyext/api.py Log: Add PyBaseString_Type Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri Apr 30 10:43:29 2010 @@ -290,6 +290,7 @@ "Type": "space.w_type", "String": "space.w_str", "Unicode": "space.w_unicode", + "BaseString": "space.w_basestring", "Dict": "space.w_dict", "Tuple": "space.w_tuple", "List": "space.w_list", From arigo at codespeak.net Fri Apr 30 11:25:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 11:25:52 +0200 (CEST) Subject: [pypy-svn] r74275 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100430092552.C2443282BF1@codespeak.net> Author: arigo Date: Fri Apr 30 11:25:51 2010 New Revision: 74275 Removed: pypy/branch/blackhole-improvement/pypy/jit/metainterp/dump.py Log: Delete unused module. From jandem at codespeak.net Fri Apr 30 11:57:20 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 11:57:20 +0200 (CEST) Subject: [pypy-svn] r74276 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100430095720.C8DA2282BF1@codespeak.net> Author: jandem Date: Fri Apr 30 11:57:19 2010 New Revision: 74276 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_arraymodule.py Log: Add wrapper for tp_iternext Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri Apr 30 11:57:19 2010 @@ -768,15 +768,20 @@ @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True)(space, func, *args) + return make_generic_cpy_call(FT, True, False)(space, func, *args) @specialize.ll() def generic_cpy_call_dont_decref(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, False)(space, func, *args) + return make_generic_cpy_call(FT, False, False)(space, func, *args) + + at specialize.ll() +def generic_cpy_call_expect_null(space, func, *args): + FT = lltype.typeOf(func).TO + return make_generic_cpy_call(FT, True, True)(space, func, *args) @specialize.memo() -def make_generic_cpy_call(FT, decref_args): +def make_generic_cpy_call(FT, decref_args, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) @@ -843,7 +848,7 @@ if has_error and has_result: raise OperationError(space.w_SystemError, space.wrap( "An exception was set, but function returned a value")) - elif not has_error and not has_result: + elif not expect_null and not has_error and not has_result: raise OperationError(space.w_SystemError, space.wrap( "Function returned a NULL result without setting an exception")) Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri Apr 30 11:57:19 2010 @@ -5,8 +5,9 @@ PyObject from pypy.module.cpyext.typeobjectdefs import unaryfunc, wrapperfunc,\ ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, lenfunc,\ - ssizeargfunc, ssizessizeargfunc, ssizeobjargproc + ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc from pypy.module.cpyext.pyobject import from_ref +from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rlib.unroll import unrolling_iterable @@ -89,6 +90,15 @@ end = space.int_w(args_w[1]) return generic_cpy_call(space, func_target, w_self, start, end) +def wrap_next(space, w_self, w_args, func): + from pypy.module.cpyext.api import generic_cpy_call_expect_null + func_target = rffi.cast(iternextfunc, func) + check_num_args(space, w_args, 0) + w_res = generic_cpy_call_expect_null(space, func_target, w_self) + if not w_res and not PyErr_Occurred(space): + raise OperationError(space.w_StopIteration, space.w_None) + return w_res + @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=True) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check Modified: pypy/trunk/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_arraymodule.py Fri Apr 30 11:57:19 2010 @@ -15,6 +15,14 @@ assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + def test_iter(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3]) + sum = 0 + for i in arr: + sum += i + assert sum == 6 + def test_index(self): module = self.import_module(name='array') arr = module.array('i', [1,2,3,4]) From jandem at codespeak.net Fri Apr 30 13:24:00 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 13:24:00 +0200 (CEST) Subject: [pypy-svn] r74277 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20100430112400.CF3BB282BF1@codespeak.net> Author: jandem Date: Fri Apr 30 13:23:59 2010 New Revision: 74277 Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py pypy/trunk/pypy/module/cpyext/unicodeobject.py Log: Add PyUnicode_AsUTF8String Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py Fri Apr 30 13:23:59 2010 @@ -58,6 +58,12 @@ assert rffi.wcharp2unicode(buf) == 'a' rffi.free_wcharp(buf) + def test_AsUTF8String(self, space, api): + w_u = space.wrap(u'sp?m') + w_res = api.PyUnicode_AsUTF8String(w_u) + assert space.type(w_res) is space.w_str + assert space.unwrap(w_res) == 'sp\xc3\xa4m' + def test_IS(self, space, api): for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x85, 0xa0, 0x1680, 0x2000, 0x2001, 0x2002, Modified: pypy/trunk/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/unicodeobject.py Fri Apr 30 13:23:59 2010 @@ -277,6 +277,15 @@ space.wrap("decoding Unicode is not supported")) return space.call_function(w_meth, w_encoding, w_errors) + at cpython_api([PyObject], PyObject) +def PyUnicode_AsUTF8String(space, w_unicode): + """Encode a Unicode object using UTF-8 and return the result as Python string + object. Error handling is "strict". Return NULL if an exception was raised + by the codec.""" + if not PyUnicode_Check(space, w_unicode): + PyErr_BadArgument(space) + return unicodetype.encode_object(space, w_unicode, "utf-8", "strict") + if sys.platform == 'win32': @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject) def PyUnicode_EncodeMBCS(space, wchar_p, length, errors): From jandem at codespeak.net Fri Apr 30 13:44:37 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 13:44:37 +0200 (CEST) Subject: [pypy-svn] r74278 - pypy/trunk/pypy/module/cpyext Message-ID: <20100430114437.CB1D0282BF1@codespeak.net> Author: jandem Date: Fri Apr 30 13:44:36 2010 New Revision: 74278 Modified: pypy/trunk/pypy/module/cpyext/typeobject.py Log: Initialize tp_alloc slot Modified: pypy/trunk/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/typeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/typeobject.py Fri Apr 30 13:44:36 2010 @@ -460,6 +460,8 @@ pto.c_tp_flags = Py_TPFLAGS_HEAPTYPE pto.c_tp_free = llhelper(PyObject_Del.api_func.functype, PyObject_Del.api_func.get_wrapper(space)) + pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype, + PyType_GenericAlloc.api_func.get_wrapper(space)) pto.c_tp_name = rffi.str2charp(w_type.getname(space, "?")) pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out pto.c_tp_itemsize = 0 @@ -532,6 +534,8 @@ base = rffi.cast(PyTypeObjectPtr, base_pyo) if not pto.c_tp_dealloc: pto.c_tp_dealloc = base.c_tp_dealloc + if not pto.c_tp_alloc: + pto.c_tp_alloc = base.c_tp_alloc # XXX check for correct GC flags! if not pto.c_tp_free: pto.c_tp_free = base.c_tp_free From jandem at codespeak.net Fri Apr 30 14:10:56 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 14:10:56 +0200 (CEST) Subject: [pypy-svn] r74279 - in pypy/trunk/pypy: module/cpyext/src rpython/lltypesystem Message-ID: <20100430121056.21AF9282BD8@codespeak.net> Author: jandem Date: Fri Apr 30 14:10:54 2010 New Revision: 74279 Modified: pypy/trunk/pypy/module/cpyext/src/getargs.c pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: Enable 'd' en 'f' arguments in PyArg_Parse* Modified: pypy/trunk/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/src/getargs.c (original) +++ pypy/trunk/pypy/module/cpyext/src/getargs.c Fri Apr 30 14:10:54 2010 @@ -756,8 +756,6 @@ #endif // HAVE_LONG_LONG case 'f': {/* float */ - Py_FatalError("'f' unimplemented for PyArg_*\n"); -#if 0 float *p = va_arg(*p_va, float *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) @@ -765,12 +763,9 @@ else *p = (float) dval; break; -#endif } case 'd': {/* double */ - Py_FatalError("'d' unimplemented for PyArg_*\n"); -#if 0 double *p = va_arg(*p_va, double *); double dval = PyFloat_AsDouble(arg); if (PyErr_Occurred()) @@ -778,7 +773,6 @@ else *p = dval; break; -#endif } #ifndef WITHOUT_COMPLEX Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Fri Apr 30 14:10:54 2010 @@ -1002,7 +1002,7 @@ elif isinstance(cvalue, (str, unicode)): cvalue = ord(cvalue) # character -> integer - if not isinstance(cvalue, (int, long)): + if not isinstance(cvalue, (int, long, float)): raise NotImplementedError("casting %r to %r" % (TYPE1, RESTYPE)) if isinstance(RESTYPE, lltype.Ptr): From arigo at codespeak.net Fri Apr 30 14:17:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 14:17:00 +0200 (CEST) Subject: [pypy-svn] r74280 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100430121700.CC368282BD8@codespeak.net> Author: arigo Date: Fri Apr 30 14:16:59 2010 New Revision: 74280 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Don't need to check that we got OverflowError if int_xxx_ovf raises. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/flatten.py Fri Apr 30 14:16:59 2010 @@ -155,12 +155,16 @@ elif block.exitswitch is c_last_exception: # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. + lastopname = block.operations[-1].opname assert block.exits[0].exitcase is None # is this always True? self.emitline('catch_exception', TLabel(block.exits[0])) self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) for link in block.exits[1:]: - if link.exitcase is Exception: + if (link.exitcase is Exception or + (link.exitcase is OverflowError and + lastopname.startswith('int_') and + lastopname.endswith('_ovf'))): # this link captures all exceptions self.make_exception_link(link) break Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri Apr 30 14:16:59 2010 @@ -399,8 +399,5 @@ catch_exception L1 int_return %i2 L1: - goto_if_exception_mismatch $<* struct object_vtable>, L2 int_return $42 - L2: - reraise """, transform=True) From arigo at codespeak.net Fri Apr 30 14:51:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 14:51:59 +0200 (CEST) Subject: [pypy-svn] r74281 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100430125159.B2A86282BF1@codespeak.net> Author: arigo Date: Fri Apr 30 14:51:57 2010 New Revision: 74281 Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (contents, props changed) pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (contents, props changed) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Log: Add a liveness analysis step. For now it works by inserting pseudo-operations called '-live-' in the control flow graph. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/format.py Fri Apr 30 14:51:57 2010 @@ -55,8 +55,30 @@ else: print >> output, asm[0], if len(asm) > 1: - print >> output, ', '.join(map(repr, asm[1:])) + lst = map(repr, asm[1:]) + if asm[0] == '-live-': lst.sort() + print >> output, ', '.join(lst) else: print >> output res = output.getvalue() return res + +def assert_format(ssarepr, expected): + asm = format_assembler(ssarepr) + expected = str(py.code.Source(expected)).strip() + '\n' + asmlines = asm.split("\n") + explines = expected.split("\n") + for asm, exp in zip(asmlines, explines): + if asm != exp: + print + print "Got: " + asm + print "Expected: " + exp + lgt = 0 + for i in range(len(asm)): + if exp[i] == asm[i]: + lgt += 1 + else: + break + print " " + " " * lgt + "^^^^" + raise AssertionError + assert len(asmlines) == len(explines) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Fri Apr 30 14:51:57 2010 @@ -0,0 +1,30 @@ +from pypy.objspace.flow.model import Variable, SpaceOperation + + +DEFAULT_OPNAMES_REQUIRING_LIVENESS = set([ + ]) + +def compute_liveness(graph, opnames_requiring_liveness= + DEFAULT_OPNAMES_REQUIRING_LIVENESS): + for block in graph.iterblocks(): + alive = set() + for link in block.exits: + for v in link.args: + alive.add(v) + if isinstance(block.exitswitch, tuple): + for v in block.exitswitch[1:]: + alive.add(v) + else: + alive.add(v) + for i in range(len(block.operations)-1, -1, -1): + op = block.operations[i] + try: + alive.remove(op.result) + except KeyError: + pass + if op.opname in opnames_requiring_liveness: + livevars = [v for v in alive if isinstance(v, Variable)] + block.operations.insert(i, SpaceOperation('-live-', livevars, + None)) + for v in op.args: + alive.add(v) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_flatten.py Fri Apr 30 14:51:57 2010 @@ -2,7 +2,7 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.flatten import flatten_graph, reorder_renaming_list from pypy.jit.codewriter.flatten import GraphFlattener, ListOfKind, Register -from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.format import assert_format from pypy.jit.codewriter.jitter import transform_graph from pypy.jit.metainterp.history import AbstractDescr from pypy.rpython.lltypesystem import lltype, rclass, rstr @@ -77,27 +77,7 @@ if transform: transform_graph(graphs[0], FakeCPU()) ssarepr = flatten_graph(graphs[0], fake_regallocs()) - self.assert_format(ssarepr, expected) - - def assert_format(self, ssarepr, expected): - asm = format_assembler(ssarepr) - expected = str(py.code.Source(expected)).strip() + '\n' - asmlines = asm.split("\n") - explines = expected.split("\n") - for asm, exp in zip(asmlines, explines): - if asm != exp: - print - print "Got: " + asm - print "Expected: " + exp - lgt = 0 - for i in range(len(asm)): - if exp[i] == asm[i]: - lgt += 1 - else: - break - print " " + " " * lgt + "^^^^" - raise AssertionError - assert len(asmlines) == len(explines) + assert_format(ssarepr, expected) def test_simple(self): def f(n): @@ -176,7 +156,7 @@ v5) # result flattener = GraphFlattener(None, fake_regallocs()) flattener.serialize_op(op) - self.assert_format(flattener.ssarepr, """ + assert_format(flattener.ssarepr, """ residual_call_ir_f $12345, I[%i0, %i1], R[%r0, %r1], %f0 """) @@ -196,7 +176,7 @@ op = SpaceOperation('foobar', [FooDescr()], None) flattener = GraphFlattener(None, fake_regallocs()) flattener.serialize_op(op) - self.assert_format(flattener.ssarepr, """ + assert_format(flattener.ssarepr, """ foobar hi_there! """) Added: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- (empty file) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Fri Apr 30 14:51:57 2010 @@ -0,0 +1,108 @@ +from pypy.jit.codewriter import support +from pypy.jit.codewriter.liveness import compute_liveness +from pypy.jit.codewriter.test.test_flatten import fake_regallocs +from pypy.jit.codewriter.flatten import flatten_graph +from pypy.jit.codewriter.format import assert_format + + +class TestFlatten: + + def make_graphs(self, func, values, type_system='lltype'): + self.rtyper = support.annotate(func, values, type_system=type_system) + return self.rtyper.annotator.translator.graphs + + def encoding_test(self, func, args, expected): + graphs = self.make_graphs(func, args) + compute_liveness(graphs[0], ['int_add', 'int_mul']) + ssarepr = flatten_graph(graphs[0], fake_regallocs()) + assert_format(ssarepr, expected) + + def test_simple_no_live(self): + def f(n): + return n + 10 + self.encoding_test(f, [5], """ + -live- + int_add %i0, $10, %i1 + int_return %i1 + """) + + def test_simple(self): + def f(n): + return (n + 10) * (n + 3) * (n + 6) + self.encoding_test(f, [5], """ + -live- %i0 + int_add %i0, $10, %i1 + -live- %i0, %i1 + int_add %i0, $3, %i2 + -live- %i0 + int_mul %i1, %i2, %i3 + -live- %i3 + int_add %i0, $6, %i4 + -live- + int_mul %i3, %i4, %i5 + int_return %i5 + """) + + def test_one_path(self): + def f(x, y): + if x+5: + return x+1 + return y+2 + self.encoding_test(f, [5, 6], """ + -live- %i0, %i1 + int_add %i0, $5, %i2 + int_is_true %i2, %i3 + goto_if_not L1, %i3 + int_copy %i0, %i4 + -live- + int_add %i4, $1, %i5 + int_return %i5 + L1: + int_copy %i1, %i6 + -live- + int_add %i6, $2, %i7 + int_return %i7 + """) + + def test_other_path(self): + def f(x, y): + if x+5: + return x+y + return x+2 + self.encoding_test(f, [5, 6], """ + -live- %i0, %i1 + int_add %i0, $5, %i2 + int_is_true %i2, %i3 + goto_if_not L1, %i3 + int_copy %i0, %i4 + int_copy %i1, %i5 + -live- + int_add %i4, %i5, %i6 + int_return %i6 + L1: + int_copy %i0, %i7 + -live- + int_add %i7, $2, %i8 + int_return %i8 + """) + + def test_no_path(self): + def f(x, y): + if x+y: + return x+5 + return x+2 + self.encoding_test(f, [5, 6], """ + -live- %i0 + int_add %i0, %i1, %i2 + int_is_true %i2, %i3 + goto_if_not L1, %i3 + int_copy %i0, %i4 + -live- + int_add %i4, $5, %i5 + int_return %i5 + L1: + int_copy %i0, %i6 + -live- + int_add %i6, $2, %i7 + int_return %i7 + """) From arigo at codespeak.net Fri Apr 30 15:07:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 15:07:09 +0200 (CEST) Subject: [pypy-svn] r74282 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100430130709.BFE50282BF1@codespeak.net> Author: arigo Date: Fri Apr 30 15:07:08 2010 New Revision: 74282 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Log: Store the liveness info in parallel with the actual bytecode. For now it's stored in a dict; later we might need to compactify this a bit. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Fri Apr 30 15:07:08 2010 @@ -11,12 +11,12 @@ _empty_f = [] def __init__(self, name, cfnptr=None, calldescr=None, called_from=None, - graph=None): + liveness=None): self.name = name - self.cfnptr = cfnptr - self.calldescr = calldescr - self.called_from = called_from - self.graph = graph + #self.cfnptr = cfnptr + #self.calldescr = calldescr + #self.called_from = called_from + self.liveness = liveness def setup(self, code, constants_i=[], constants_r=[], constants_f=[], num_regs_r=256): @@ -37,6 +37,36 @@ def highest_r_reg(self): return ord(self.code[-1]) + def enumerate_live_vars(self, pc, callback, arg, + registers_i, registers_r, registers_f): + # 'pc' gives a position in this bytecode. This invokes + # 'callback' for each variable that is live across the + # instruction which starts at 'pc'. (It excludes the arguments + # of that instruction which are no longer used afterwards, and + # also the return value of that instruction.) More precisely, + # this invokes 'callback(arg, box)' where 'box' comes from one + # of the three lists of registers. + live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! + for c in live_i: callback(arg, registers_i[ord(c)]) + for c in live_r: callback(arg, registers_r[ord(c)]) + for c in live_f: callback(arg, registers_f[ord(c)]) + enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' + + def _live_vars(self, pc): + # for testing only + class Names: + def __init__(self, kind): + self.kind = kind + def __getitem__(self, index): + return '%%%s%d' % (self.kind, index) + def callback(lst, reg): + lst.append(reg) + lst = [] + self.enumerate_live_vars(pc, callback, lst, + Names('i'), Names('r'), Names('f')) + lst.sort() + return ' '.join(lst) + class Assembler(object): @@ -63,6 +93,7 @@ self.tlabel_positions = [] self.switchdictdescrs = [] self.count_regs = dict.fromkeys(KINDS, 0) + self.liveness = {} def emit_reg(self, reg): if reg.index >= self.count_regs[reg.kind]: @@ -106,6 +137,12 @@ if isinstance(insn[0], Label): self.label_positions[insn[0].name] = len(self.code) return + if insn[0] == '-live-': + self.liveness[len(self.code)] = ( + self.get_liveness_info(insn, 'int'), + self.get_liveness_info(insn, 'ref'), + self.get_liveness_info(insn, 'float')) + return startposition = len(self.code) self.code.append("temporary placeholder") # @@ -160,6 +197,11 @@ num = self.insns.setdefault(key, len(self.insns)) self.code[startposition] = chr(num) + def get_liveness_info(self, insn, kind): + lives = [chr(reg.index) for reg in insn[1:] if reg.kind == kind] + lives.sort() + return ''.join(lives) + def fix_labels(self): for name, pos in self.tlabel_positions: assert self.code[pos ] == "temp 1" @@ -181,7 +223,7 @@ assert self.count_regs['float'] + len(self.constants_f) <= 256 def make_jitcode(self, name): - jitcode = JitCode(name) + jitcode = JitCode(name, liveness=self.liveness) jitcode.setup(''.join(self.code), self.constants_i, self.constants_r, Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Fri Apr 30 15:07:08 2010 @@ -1,4 +1,4 @@ -import struct +import py, struct from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.flatten import SSARepr, Label, TLabel, Register from pypy.jit.codewriter.flatten import ListOfKind @@ -150,3 +150,40 @@ ssarepr.insns = [('foobar', Register('int', 51), Register('ref', 27))] jitcode = assembler.assemble(ssarepr) assert jitcode.highest_r_reg() == 27 + +def test_liveness(): + ssarepr = SSARepr("test") + i0, i1, i2 = Register('int', 0), Register('int', 1), Register('int', 2) + i3, i4, i5 = Register('int', 3), Register('int', 4), Register('int', 5) + ssarepr.insns = [ + ('-live-', i0), + ('int_add', i0, Constant(10, lltype.Signed), i1), + ('-live-', i0, i1), + ('int_add', i0, Constant(3, lltype.Signed), i2), + ('-live-', i0), + ('int_mul', i1, i2, i3), + ('-live-', i3), + ('int_add', i0, Constant(6, lltype.Signed), i4), + ('-live-',), + ('int_mul', i3, i4, i5), + ('int_return', i5), + ] + assembler = Assembler() + jitcode = assembler.assemble(ssarepr) + assert jitcode._code() == ("\x00\x00\x0A\x01" + "\x00\x00\x03\x02" + "\x01\x01\x02\x03" + "\x00\x00\x06\x04" + "\x01\x03\x04\x05" + "\x02\x05") + assert assembler.insns == {'int_add/ici': 0, + 'int_mul/iii': 1, + 'int_return/i': 2} + py.test.raises(KeyError, jitcode._live_vars, 1) + py.test.raises(KeyError, jitcode._live_vars, 3) + py.test.raises(KeyError, jitcode._live_vars, 20) + assert jitcode._live_vars(0) == '%i0' + assert jitcode._live_vars(4) == '%i0 %i1' + assert jitcode._live_vars(8) == '%i0' + assert jitcode._live_vars(12) == '%i3' + assert jitcode._live_vars(16) == '' From arigo at codespeak.net Fri Apr 30 15:29:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 15:29:27 +0200 (CEST) Subject: [pypy-svn] r74283 - in pypy/branch/blackhole-improvement/pypy/jit/codewriter: . test Message-ID: <20100430132927.A7857282BD8@codespeak.net> Author: arigo Date: Fri Apr 30 15:29:25 2010 New Revision: 74283 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Log: Integration tests and fixes for 'liveness'. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/codewriter.py Fri Apr 30 15:29:25 2010 @@ -4,6 +4,7 @@ from pypy.jit.codewriter.assembler import Assembler from pypy.jit.codewriter.jitter import transform_graph from pypy.jit.codewriter.format import format_assembler +from pypy.jit.codewriter.liveness import compute_liveness class CodeWriter(object): @@ -28,11 +29,14 @@ # that we want in the JitCode, but still as a control flow graph transform_graph(graph, self.cpu) # - # step 2: perform register allocation on it + # step 2a: perform register allocation on it regallocs = {} for kind in KINDS: regallocs[kind] = perform_register_allocation(graph, kind) # + # step 2b: compute the liveness around certain operations + compute_liveness(graph) + # # step 3: flatten the graph to produce human-readable "assembler", # which means mostly producing a linear list of operations and # inserting jumps or conditional jumps. This is a list of tuples Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Fri Apr 30 15:29:25 2010 @@ -1,30 +1,51 @@ +import re from pypy.objspace.flow.model import Variable, SpaceOperation -DEFAULT_OPNAMES_REQUIRING_LIVENESS = set([ - ]) +def compile_re(opnames): + return re.compile('|'.join(opnames)) -def compute_liveness(graph, opnames_requiring_liveness= - DEFAULT_OPNAMES_REQUIRING_LIVENESS): +# List of instruction prefixes that might need the liveness information +# (they are the ones that can end up in generate_guard() in pyjitpl.py). +# Note that the goto_if_not_* operations are not present in the control +# flow graph format; their liveness information is attached by setting +# 'switches_require_liveness' to True. +DEFAULT_OPNAMES_REQUIRING_LIVENESS = [ + 'residual_call_', + ] + +# ____________________________________________________________ + +def compute_liveness(graph, + switches_require_liveness = True, + opnames_requiring_liveness = + compile_re(DEFAULT_OPNAMES_REQUIRING_LIVENESS)): + if isinstance(opnames_requiring_liveness, list): + opnames_requiring_liveness = compile_re(opnames_requiring_liveness) for block in graph.iterblocks(): + num_operations = len(block.operations) alive = set() for link in block.exits: for v in link.args: alive.add(v) + if switches_require_liveness and len(block.exits) > 1: + block.operations.append(_livespaceop(alive)) if isinstance(block.exitswitch, tuple): for v in block.exitswitch[1:]: alive.add(v) else: alive.add(v) - for i in range(len(block.operations)-1, -1, -1): + for i in range(num_operations-1, -1, -1): op = block.operations[i] try: alive.remove(op.result) except KeyError: pass - if op.opname in opnames_requiring_liveness: - livevars = [v for v in alive if isinstance(v, Variable)] - block.operations.insert(i, SpaceOperation('-live-', livevars, - None)) + if opnames_requiring_liveness.match(op.opname): + block.operations.insert(i, _livespaceop(alive)) for v in op.args: alive.add(v) + +def _livespaceop(alive): + livevars = [v for v in alive if isinstance(v, Variable)] + return SpaceOperation('-live-', livevars, None) Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py Fri Apr 30 15:29:25 2010 @@ -1,3 +1,4 @@ +import py from pypy.jit.codewriter.codewriter import CodeWriter @@ -19,6 +20,9 @@ 'int_sub/ici': 2, 'goto/L': 3, 'int_return/i': 4} + assert jitcode._live_vars(0) == '%i0 %i1' + for i in range(1, len(jitcode._code())): + py.test.raises(KeyError, jitcode._live_vars, i) def test_integration(): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_liveness.py Fri Apr 30 15:29:25 2010 @@ -11,9 +11,12 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs - def encoding_test(self, func, args, expected): + def encoding_test(self, func, args, expected, + switches_require_liveness=False): graphs = self.make_graphs(func, args) - compute_liveness(graphs[0], ['int_add', 'int_mul']) + compute_liveness(graphs[0], + switches_require_liveness=switches_require_liveness, + opnames_requiring_liveness=['int_add', 'int_mul']) ssarepr = flatten_graph(graphs[0], fake_regallocs()) assert_format(ssarepr, expected) @@ -106,3 +109,17 @@ int_add %i6, $2, %i7 int_return %i7 """) + + def test_switch_require_liveness(self): + def f(x, y): + if x: + return x + return y + self.encoding_test(f, [5, 6], """ + int_is_true %i0, %i2 + -live- %i0, %i1 + goto_if_not L1, %i2 + int_return %i0 + L1: + int_return %i1 + """, switches_require_liveness=True) From jandem at codespeak.net Fri Apr 30 15:34:51 2010 From: jandem at codespeak.net (jandem at codespeak.net) Date: Fri, 30 Apr 2010 15:34:51 +0200 (CEST) Subject: [pypy-svn] r74284 - in pypy/trunk/pypy/module/cpyext: . patches Message-ID: <20100430133451.16683282BD8@codespeak.net> Author: jandem Date: Fri Apr 30 15:34:50 2010 New Revision: 74284 Added: pypy/trunk/pypy/module/cpyext/patches/ pypy/trunk/pypy/module/cpyext/patches/numpy.patch - copied unchanged from r74273, pypy/trunk/pypy/module/cpyext/numpy.patch pypy/trunk/pypy/module/cpyext/patches/pycairo.patch Removed: pypy/trunk/pypy/module/cpyext/numpy.patch Log: Add patch to make pycairo work, will remove it when they accept it. Added: pypy/trunk/pypy/module/cpyext/patches/pycairo.patch ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/patches/pycairo.patch Fri Apr 30 15:34:50 2010 @@ -0,0 +1,110 @@ +diff --git a/src/cairomodule.c b/src/cairomodule.c +index 21b7b20..181b420 100644 +--- a/src/cairomodule.c ++++ b/src/cairomodule.c +@@ -188,7 +188,8 @@ init_cairo(void) + if (PyType_Ready(&PycairoPath_Type) < 0) + return; + PycairoPathiter_Type.tp_iter=&PyObject_SelfIter; +- ++ if (PyType_Ready(&PycairoPathiter_Type) < 0) ++ return; + if (PyType_Ready(&PycairoPattern_Type) < 0) + return; + if (PyType_Ready(&PycairoSolidPattern_Type) < 0) +diff --git a/src/context.c b/src/context.c +index e2c08b5..93b369f 100644 +--- a/src/context.c ++++ b/src/context.c +@@ -1430,7 +1430,7 @@ PyTypeObject PycairoContext_Type = { + pycairo_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +diff --git a/src/font.c b/src/font.c +index 97cf1b3..3fcc9e4 100644 +--- a/src/font.c ++++ b/src/font.c +@@ -131,7 +131,7 @@ PyTypeObject PycairoFontFace_Type = { + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +@@ -410,7 +410,7 @@ PyTypeObject PycairoScaledFont_Type = { + scaled_font_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +@@ -595,7 +595,7 @@ PyTypeObject PycairoFontOptions_Type = { + font_options_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +diff --git a/src/matrix.c b/src/matrix.c +index eefeab9..2ec5e2d 100644 +--- a/src/matrix.c ++++ b/src/matrix.c +@@ -332,7 +332,7 @@ PyTypeObject PycairoMatrix_Type = { + matrix_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +diff --git a/src/path.c b/src/path.c +index 97ca35c..2d1bcf5 100644 +--- a/src/path.c ++++ b/src/path.c +@@ -206,7 +206,7 @@ PyTypeObject PycairoPath_Type = { + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +diff --git a/src/pattern.c b/src/pattern.c +index 229e218..cebf0c9 100644 +--- a/src/pattern.c ++++ b/src/pattern.c +@@ -194,7 +194,7 @@ PyTypeObject PycairoPattern_Type = { + pattern_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ +diff --git a/src/surface.c b/src/surface.c +index 568aad7..fa11413 100644 +--- a/src/surface.c ++++ b/src/surface.c +@@ -370,7 +370,7 @@ PyTypeObject PycairoSurface_Type = { + surface_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ +- &PyBaseObject_Type, /* tp_base */ ++ 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ From afa at codespeak.net Fri Apr 30 15:44:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 15:44:28 +0200 (CEST) Subject: [pypy-svn] r74285 - pypy/trunk/pypy/module/cpyext Message-ID: <20100430134428.2097F282BD8@codespeak.net> Author: afa Date: Fri Apr 30 15:44:26 2010 New Revision: 74285 Modified: pypy/trunk/pypy/module/cpyext/TODO Log: A problem with PyWeakref_GetObject Modified: pypy/trunk/pypy/module/cpyext/TODO ============================================================================== --- pypy/trunk/pypy/module/cpyext/TODO (original) +++ pypy/trunk/pypy/module/cpyext/TODO Fri Apr 30 15:44:26 2010 @@ -23,6 +23,9 @@ - refactor management of py_objects_r2w and py_objects_w2r, this can probably be expressed in terms of _PyObject_GC_TRACK macros. + - PyWeakref_GetObject() returns a borrowed reference, but this turns the + WeakRef into a strong reference! + - sort out pypy's buffer protocol. PyPy's buffer right now don't support raw memory (except array which supports it in a hackish way), which should be fixed in order to make it nicely work with cpyext. From hpk at codespeak.net Fri Apr 30 16:45:09 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 16:45:09 +0200 (CEST) Subject: [pypy-svn] r74286 - in pypy/branch/py12/pypy: module/_demo/test tool/test Message-ID: <20100430144509.B1AF0282BF2@codespeak.net> Author: hpk Date: Fri Apr 30 16:45:08 2010 New Revision: 74286 Added: pypy/branch/py12/pypy/module/_demo/test/__init__.py Modified: pypy/branch/py12/pypy/tool/test/test_pytestsupport.py Log: fix missing init and wrong reinterpret call Added: pypy/branch/py12/pypy/module/_demo/test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/pypy/module/_demo/test/__init__.py Fri Apr 30 16:45:08 2010 @@ -0,0 +1 @@ +# Modified: pypy/branch/py12/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/branch/py12/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/py12/pypy/tool/test/test_pytestsupport.py Fri Apr 30 16:45:08 2010 @@ -22,8 +22,8 @@ co = PyCode._from_code(space, somefunc.func_code) pyframe = PyFrame(space, co, space.newdict(), None) runner = AppFrame(space, pyframe) - py.code.reinterpret_old("f = lambda x: x+1", runner, should_fail=False) - msg = py.code.reinterpret_old("assert isinstance(f(2), float)", runner) + py.code._reinterpret_old("f = lambda x: x+1", runner, should_fail=False) + msg = py.code._reinterpret_old("assert isinstance(f(2), float)", runner) assert msg.startswith("assert isinstance(3, float)\n" " + where 3 = ") From arigo at codespeak.net Fri Apr 30 16:47:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 16:47:04 +0200 (CEST) Subject: [pypy-svn] r74287 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter codewriter/test metainterp Message-ID: <20100430144704.1D0FF282BF2@codespeak.net> Author: arigo Date: Fri Apr 30 16:47:02 2010 New Revision: 74287 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Log: Stop abusing the last character of the bytecode, and instead store all three num_regs_X as a single separate integer. Use it to implement replace_box() efficiently. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Fri Apr 30 16:47:02 2010 @@ -3,6 +3,7 @@ from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rlib.objectmodel import we_are_translated class JitCode(AbstractValue): @@ -11,31 +12,34 @@ _empty_f = [] def __init__(self, name, cfnptr=None, calldescr=None, called_from=None, - liveness=None): + liveness=None, assembler=None): self.name = name #self.cfnptr = cfnptr #self.calldescr = calldescr #self.called_from = called_from self.liveness = liveness + self._assembler = assembler def setup(self, code, constants_i=[], constants_r=[], constants_f=[], - num_regs_r=256): - # stick an extra char at the end of self.code, which is the - # highest 'r' register used in this code. It default to 255, - # which is always correct. Also, even if no 'r' register is - # used it must be set to 0, which means that register %r0 is - # always marked as used. - self.code = code + chr((num_regs_r or 1)-1) + num_regs_i=256, num_regs_r=256, num_regs_f=256): + self.code = code # if the following lists are empty, use a single shared empty list self.constants_i = constants_i or self._empty_i self.constants_r = constants_r or self._empty_r self.constants_f = constants_f or self._empty_f + # encode the three num_regs into a single integer + self.num_regs_encoded = ((num_regs_i << 18) | + (num_regs_r << 9) | + (num_regs_f << 0)) - def _code(self): - return self.code[:-1] # for testing, without the extra char + def num_regs_i(self): + return self.num_regs_encoded >> 18 - def highest_r_reg(self): - return ord(self.code[-1]) + def num_regs_r(self): + return (self.num_regs_encoded >> 9) & 0x1FF + + def num_regs_f(self): + return self.num_regs_encoded & 0x1FF def enumerate_live_vars(self, pc, callback, arg, registers_i, registers_r, registers_f): @@ -45,11 +49,23 @@ # of that instruction which are no longer used afterwards, and # also the return value of that instruction.) More precisely, # this invokes 'callback(arg, box)' where 'box' comes from one - # of the three lists of registers. + # of the three lists of registers. If the callback returns a + # box, then it is stored back. + if not we_are_translated() and pc not in self.liveness: + self._missing_liveness(pc) live_i, live_r, live_f = self.liveness[pc] # XXX compactify!! - for c in live_i: callback(arg, registers_i[ord(c)]) - for c in live_r: callback(arg, registers_r[ord(c)]) - for c in live_f: callback(arg, registers_f[ord(c)]) + for c in live_i: + newbox = callback(arg, registers_i[ord(c)]) + if newbox is not None: + registers_i[ord(c)] = newbox + for c in live_r: + newbox = callback(arg, registers_r[ord(c)]) + if newbox is not None: + registers_r[ord(c)] = newbox + for c in live_f: + newbox = callback(arg, registers_f[ord(c)]) + if newbox is not None: + registers_f[ord(c)] = newbox enumerate_live_vars._annspecialcase_ = 'specialize:arg(2)' def _live_vars(self, pc): @@ -67,6 +83,16 @@ lst.sort() return ' '.join(lst) + def _missing_liveness(self, pc): + opcode = ord(self.code[pc]) + insn = 'insn %d' % opcode + if self._assembler is not None: + for name, code in self._assembler.insns.items(): + if code == opcode: + insn = name + raise KeyError("missing liveness[%d], corresponding to %r" % ( + pc, insn)) + class Assembler(object): @@ -223,10 +249,13 @@ assert self.count_regs['float'] + len(self.constants_f) <= 256 def make_jitcode(self, name): - jitcode = JitCode(name, liveness=self.liveness) + jitcode = JitCode(name, liveness=self.liveness, + assembler=self) jitcode.setup(''.join(self.code), self.constants_i, self.constants_r, self.constants_f, - self.count_regs['ref']) + self.count_regs['int'], + self.count_regs['ref'], + self.count_regs['float']) return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/liveness.py Fri Apr 30 16:47:02 2010 @@ -12,6 +12,7 @@ # 'switches_require_liveness' to True. DEFAULT_OPNAMES_REQUIRING_LIVENESS = [ 'residual_call_', + '(int|ref|float)_guard_value', ] # ____________________________________________________________ Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/test/test_assembler.py Fri Apr 30 16:47:02 2010 @@ -17,10 +17,12 @@ assembler = Assembler() jitcode = assembler.assemble(ssarepr) assert jitcode.code == ("\x00\x00\x01\x02" - "\x01\x02" - "\x00") + "\x01\x02") assert assembler.insns == {'int_add/iii': 0, 'int_return/i': 1} + assert jitcode.num_regs_i() == 3 + assert jitcode.num_regs_r() == 0 + assert jitcode.num_regs_f() == 0 def test_assemble_consts(): ssarepr = SSARepr("test") @@ -37,8 +39,7 @@ "\x01\x12" # use int_return/c for one-byte consts "\x01\xFC" "\x00\xFF" # use int_return/i for larger consts - "\x00\xFE" - "\x00") # highest_r_reg + "\x00\xFE") assert assembler.insns == {'int_return/i': 0, 'int_return/c': 1} assert jitcode.constants_i == [128, -129] @@ -56,8 +57,7 @@ assert jitcode.code == ("\x00\x0D" "\x00\xFF" "\x00\xFE" - "\x00\xFD" - "\x00") + "\x00\xFD") assert assembler.insns == {'float_return/f': 0} assert jitcode.constants_f == [18.0, -4.0, 128.1] @@ -78,8 +78,7 @@ assert jitcode.code == ("\x00\x58" "\x01\xFF" "\x01\xFE" - "\x02\xFF" - "\x00") + "\x02\xFF") assert assembler.insns == {'int_return/c': 0, 'int_return/i': 1, 'ref_return/r': 2} @@ -106,8 +105,7 @@ "\x01\x17\x16\x17" "\x02\x16\x01\x16" "\x03\x00\x00" - "\x04\x17" - "\x00") + "\x04\x17") assert assembler.insns == {'goto_if_not_int_gt/Lic': 0, 'int_add/iii': 1, 'int_sub/ici': 2, @@ -123,7 +121,7 @@ ] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode._code() == "\x00\x03\x16\x17\xFF\x00" + assert jitcode.code == "\x00\x03\x16\x17\xFF\x00" assert assembler.insns == {'foobar/IR': 0} assert jitcode.constants_i == [42] @@ -135,21 +133,27 @@ ssarepr.insns = [('foobar', d) for d in descrs[::-1]] assembler = Assembler() jitcode = assembler.assemble(ssarepr) - assert jitcode._code() == ''.join(["\x00" + struct.pack("= 0: + for i in range(self.cleanup_required_in_registers_r): self.registers_r[i] = NULL - i -= 1 + self.cleanup_required_in_registers_r = 0 self.tmpreg_r = NULL self.exception_last_value = None Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri Apr 30 16:47:02 2010 @@ -93,77 +93,36 @@ outvalue[startindex+i] = reg prepare_list_of_boxes._annspecialcase_ = 'specialize:arg(4)' - def load_int(self): - pc = self.pc - result = ord(self.bytecode[pc]) - self.pc = pc + 1 - if result > 0x7F: - result = self._load_larger_int(result) - return result - - def _load_larger_int(self, result): # slow path - result = result & 0x7F - shift = 7 - pc = self.pc - while 1: - byte = ord(self.bytecode[pc]) - pc += 1 - result += (byte & 0x7F) << shift - shift += 7 - if not byte & 0x80: - break - self.pc = pc - return intmask(result) - _load_larger_int._dont_inline_ = True - - def load_3byte(self): - pc = self.pc - result = (((ord(self.bytecode[pc + 0])) << 16) | - ((ord(self.bytecode[pc + 1])) << 8) | - ((ord(self.bytecode[pc + 2])) << 0)) - self.pc = pc + 3 - return result - - def load_bool(self): - pc = self.pc - result = ord(self.bytecode[pc]) - self.pc = pc + 1 - return bool(result) - - def getenv(self, i): - assert i >= 0 - j = i >> 1 - if i & 1: - return self.constants[j] - else: - assert j < len(self.env) - return self.env[j] - - def load_arg(self): - return self.getenv(self.load_int()) - - def load_const_arg(self): - return self.constants[self.load_int()] - - def load_varargs(self): - count = self.load_int() - return [self.load_arg() for i in range(count)] - - def load_constargs(self): - count = self.load_int() - return [self.load_const_arg() for i in range(count)] - - def ignore_varargs(self): - count = self.load_int() + def get_list_of_active_boxes(self): + # XXX find a way to avoid needing the temporary 'env' as a + # variable-sized list + env = [] + self.jitcode.enumerate_live_vars( + self.pc, MIFrame._store_in_env, env, + self.registers_i, self.registers_r, self.registers_f) + return env[:] + + @staticmethod + def _store_in_env(env, box): + env.append(box) + + def replace_active_box_in_frame(self, oldbox, newbox): + if isinstance(oldbox, history.BoxInt): + count = self.jitcode.num_regs_i() + registers = self.registers_i + elif isinstance(oldbox, history.BoxPtr): + count = self.jitcode.num_regs_r() + registers = self.registers_r + elif isinstance(oldbox, history.BoxFloat): + count = self.jitcode.num_regs_f() + registers = self.registers_f + else: + assert 0, repr(oldbox) for i in range(count): - self.load_int() - - def getvarenv(self, i): - return self.env[i] - - def make_result_box(self, box): - assert isinstance(box, Box) or isinstance(box, Const) - self.env.append(box) + if registers[i] is oldbox: + registers[i] = newbox + if not we_are_translated(): + assert oldbox not in registers[count:] # ------------------------------ @@ -1999,11 +1958,9 @@ return boxes def replace_box(self, oldbox, newbox): + assert isinstance(oldbox, Box) for frame in self.framestack: - boxes = frame.env - for i in range(len(boxes)): - if boxes[i] is oldbox: - boxes[i] = newbox + frame.replace_active_box_in_frame(oldbox, newbox) boxes = self.virtualref_boxes for i in range(len(boxes)): if boxes[i] is oldbox: Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/resume.py Fri Apr 30 16:47:02 2010 @@ -21,13 +21,12 @@ self.boxes = boxes class FrameInfo(object): - __slots__ = ('prev', 'jitcode', 'pc', 'exception_target') + __slots__ = ('prev', 'jitcode', 'pc') def __init__(self, prev, frame): self.prev = prev self.jitcode = frame.jitcode self.pc = frame.pc - self.exception_target = frame.exception_target def _ensure_parent_resumedata(framestack, n): target = framestack[n] @@ -40,7 +39,7 @@ back) target.parent_resumedata_snapshot = Snapshot( back.parent_resumedata_snapshot, - back.env[:]) + back.get_list_of_active_boxes()) def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, storage): @@ -50,7 +49,8 @@ frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list, top) storage.rd_frame_info_list = frame_info_list - snapshot = Snapshot(top.parent_resumedata_snapshot, top.env[:]) + snapshot = Snapshot(top.parent_resumedata_snapshot, + top.get_list_of_active_boxes()) snapshot = Snapshot(snapshot, virtualref_boxes[:]) # xxx for now if virtualizable_boxes is not None: snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now @@ -576,7 +576,7 @@ except AttributeError: jitcodename = str(compute_unique_id(frameinfo.jitcode)) debug_print('\tjitcode/pc', jitcodename, - frameinfo.pc, frameinfo.exception_target, + frameinfo.pc, 'at', compute_unique_id(frameinfo)) frameinfo = frameinfo.prev numb = storage.rd_numb From hpk at codespeak.net Fri Apr 30 16:50:32 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 16:50:32 +0200 (CEST) Subject: [pypy-svn] r74288 - pypy/build/testrunner Message-ID: <20100430145032.DDE4A282BF2@codespeak.net> Author: hpk Date: Fri Apr 30 16:50:31 2010 New Revision: 74288 Modified: pypy/build/testrunner/runner.py Log: make sure the testrunner requests the resultlog plugin before using the --resultlog option. Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Fri Apr 30 16:50:31 2010 @@ -97,7 +97,7 @@ do_dry_run=False, timeout=None, _win32=(sys.platform=='win32')): args = interp + test_driver - args += ['--resultlog=%s' % logfname, test] + args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test] args = map(str, args) interp0 = args[0] From hpk at codespeak.net Fri Apr 30 17:07:56 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 17:07:56 +0200 (CEST) Subject: [pypy-svn] r74289 - in pypy/branch/py12/py: . _cmdline _code _compat _io _log _path _path/gateway _plugin _process _test Message-ID: <20100430150756.867CC282BF2@codespeak.net> Author: hpk Date: Fri Apr 30 17:07:52 2010 New Revision: 74289 Added: pypy/branch/py12/py/ pypy/branch/py12/py/__init__.py pypy/branch/py12/py/__init__.py.orig pypy/branch/py12/py/__metainfo.py pypy/branch/py12/py/_builtin.py pypy/branch/py12/py/_cmdline/ pypy/branch/py12/py/_cmdline/__init__.py pypy/branch/py12/py/_cmdline/pycleanup.py (contents, props changed) pypy/branch/py12/py/_cmdline/pyconvert_unittest.py pypy/branch/py12/py/_cmdline/pycountloc.py (contents, props changed) pypy/branch/py12/py/_cmdline/pylookup.py (contents, props changed) pypy/branch/py12/py/_cmdline/pysvnwcrevert.py (contents, props changed) pypy/branch/py12/py/_cmdline/pytest.py (contents, props changed) pypy/branch/py12/py/_cmdline/pywhich.py (contents, props changed) pypy/branch/py12/py/_code/ pypy/branch/py12/py/_code/__init__.py pypy/branch/py12/py/_code/_assertionnew.py pypy/branch/py12/py/_code/_assertionold.py pypy/branch/py12/py/_code/assertion.py pypy/branch/py12/py/_code/code.py pypy/branch/py12/py/_code/oldmagic.py pypy/branch/py12/py/_code/oldmagic2.py pypy/branch/py12/py/_code/source.py pypy/branch/py12/py/_compat/ pypy/branch/py12/py/_compat/__init__.py pypy/branch/py12/py/_compat/dep_doctest.py pypy/branch/py12/py/_compat/dep_optparse.py pypy/branch/py12/py/_compat/dep_subprocess.py pypy/branch/py12/py/_compat/dep_textwrap.py pypy/branch/py12/py/_error.py pypy/branch/py12/py/_io/ pypy/branch/py12/py/_io/__init__.py pypy/branch/py12/py/_io/capture.py pypy/branch/py12/py/_io/saferepr.py pypy/branch/py12/py/_io/terminalwriter.py pypy/branch/py12/py/_log/ pypy/branch/py12/py/_log/__init__.py pypy/branch/py12/py/_log/log.py pypy/branch/py12/py/_log/warning.py pypy/branch/py12/py/_path/ pypy/branch/py12/py/_path/__init__.py pypy/branch/py12/py/_path/cacheutil.py pypy/branch/py12/py/_path/common.py pypy/branch/py12/py/_path/gateway/ pypy/branch/py12/py/_path/gateway/__init__.py pypy/branch/py12/py/_path/gateway/channeltest.py pypy/branch/py12/py/_path/gateway/channeltest2.py pypy/branch/py12/py/_path/gateway/remotepath.py pypy/branch/py12/py/_path/local.py pypy/branch/py12/py/_path/svnurl.py pypy/branch/py12/py/_path/svnwc.py pypy/branch/py12/py/_plugin/ pypy/branch/py12/py/_plugin/__init__.py pypy/branch/py12/py/_plugin/hookspec.py pypy/branch/py12/py/_plugin/pytest__pytest.py pypy/branch/py12/py/_plugin/pytest_assertion.py pypy/branch/py12/py/_plugin/pytest_capture.py pypy/branch/py12/py/_plugin/pytest_default.py pypy/branch/py12/py/_plugin/pytest_doctest.py pypy/branch/py12/py/_plugin/pytest_genscript.py (contents, props changed) pypy/branch/py12/py/_plugin/pytest_helpconfig.py pypy/branch/py12/py/_plugin/pytest_hooklog.py pypy/branch/py12/py/_plugin/pytest_junitxml.py pypy/branch/py12/py/_plugin/pytest_mark.py pypy/branch/py12/py/_plugin/pytest_monkeypatch.py pypy/branch/py12/py/_plugin/pytest_nose.py pypy/branch/py12/py/_plugin/pytest_pastebin.py pypy/branch/py12/py/_plugin/pytest_pdb.py pypy/branch/py12/py/_plugin/pytest_pylint.py pypy/branch/py12/py/_plugin/pytest_pytester.py pypy/branch/py12/py/_plugin/pytest_pytester.py.orig pypy/branch/py12/py/_plugin/pytest_recwarn.py pypy/branch/py12/py/_plugin/pytest_restdoc.py pypy/branch/py12/py/_plugin/pytest_resultlog.py pypy/branch/py12/py/_plugin/pytest_runner.py pypy/branch/py12/py/_plugin/pytest_skipping.py pypy/branch/py12/py/_plugin/pytest_terminal.py pypy/branch/py12/py/_plugin/pytest_terminal.py.orig pypy/branch/py12/py/_plugin/pytest_tmpdir.py pypy/branch/py12/py/_plugin/pytest_unittest.py pypy/branch/py12/py/_plugin/standalonetemplate.py (contents, props changed) pypy/branch/py12/py/_process/ pypy/branch/py12/py/_process/__init__.py pypy/branch/py12/py/_process/cmdexec.py pypy/branch/py12/py/_process/forkedfunc.py pypy/branch/py12/py/_process/killproc.py pypy/branch/py12/py/_std.py pypy/branch/py12/py/_test/ pypy/branch/py12/py/_test/.pluginmanager.py.swp (contents, props changed) pypy/branch/py12/py/_test/__init__.py pypy/branch/py12/py/_test/cmdline.py pypy/branch/py12/py/_test/collect.py pypy/branch/py12/py/_test/config.py pypy/branch/py12/py/_test/conftesthandle.py pypy/branch/py12/py/_test/funcargs.py pypy/branch/py12/py/_test/parseopt.py pypy/branch/py12/py/_test/pluginmanager.py pypy/branch/py12/py/_test/pycollect.py pypy/branch/py12/py/_test/session.py pypy/branch/py12/py/_xmlgen.py pypy/branch/py12/py/apipkg.py Log: inlining 1694:c3cb8c7b94aa snapshot of bitbucket's py-trunk repo here for now Added: pypy/branch/py12/py/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,178 @@ +""" +py.test and pylib: rapid testing and development utils + +this module uses apipkg.py for lazy-loading sub modules +and classes. The initpkg-dictionary below specifies +name->value mappings where value can be another namespace +dictionary or an import path. + +(c) Holger Krekel and others, 2004-2010 +""" +__version__ = version = "1.2.2" + +import py.apipkg + +py.apipkg.initpkg(__name__, dict( + # access to all standard lib modules + std = '._std:std', + # access to all posix errno's as classes + error = '._error:error', + + _pydir = '.__metainfo:pydir', + version = 'py:__version__', # backward compatibility + + cmdline = { + 'pytest': '._cmdline.pytest:main', + 'pylookup': '._cmdline.pylookup:main', + 'pycountloc': '._cmdline.pycountlog:main', + 'pytest': '._test.cmdline:main', + 'pylookup': '._cmdline.pylookup:main', + 'pycountloc': '._cmdline.pycountloc:main', + 'pycleanup': '._cmdline.pycleanup:main', + 'pywhich' : '._cmdline.pywhich:main', + 'pysvnwcrevert' : '._cmdline.pysvnwcrevert:main', + 'pyconvert_unittest' : '._cmdline.pyconvert_unittest:main', + }, + + test = { + # helpers for use from test functions or collectors + '__onfirstaccess__' : '._test.config:onpytestaccess', + '__doc__' : '._test:__doc__', + # configuration/initialization related test api + 'config' : '._test.config:config_per_process', + 'ensuretemp' : '._test.config:ensuretemp', + 'collect': { + 'Collector' : '._test.collect:Collector', + 'Directory' : '._test.collect:Directory', + 'File' : '._test.collect:File', + 'Item' : '._test.collect:Item', + 'Module' : '._test.pycollect:Module', + 'Class' : '._test.pycollect:Class', + 'Instance' : '._test.pycollect:Instance', + 'Generator' : '._test.pycollect:Generator', + 'Function' : '._test.pycollect:Function', + '_fillfuncargs' : '._test.funcargs:fillfuncargs', + }, + 'cmdline': { + 'main' : '._test.cmdline:main', # backward compat + }, + }, + + # hook into the top-level standard library + process = { + '__doc__' : '._process:__doc__', + 'cmdexec' : '._process.cmdexec:cmdexec', + 'kill' : '._process.killproc:kill', + 'ForkedFunc' : '._process.forkedfunc:ForkedFunc', + }, + + path = { + '__doc__' : '._path:__doc__', + 'svnwc' : '._path.svnwc:SvnWCCommandPath', + 'svnurl' : '._path.svnurl:SvnCommandPath', + 'local' : '._path.local:LocalPath', + 'SvnAuth' : '._path.svnwc:SvnAuth', + }, + + # some nice slightly magic APIs + magic = { + 'invoke' : '._code.oldmagic:invoke', + 'revoke' : '._code.oldmagic:revoke', + 'patch' : '._code.oldmagic:patch', + 'revert' : '._code.oldmagic:revert', + 'autopath' : '._path.local:autopath', + 'AssertionError' : '._code.oldmagic2:AssertionError', + }, + + # python inspection/code-generation API + code = { + '__doc__' : '._code:__doc__', + 'compile' : '._code.source:compile_', + 'Source' : '._code.source:Source', + 'Code' : '._code.code:Code', + 'Frame' : '._code.code:Frame', + 'ExceptionInfo' : '._code.code:ExceptionInfo', + 'Traceback' : '._code.code:Traceback', + 'getfslineno' : '._code.source:getfslineno', + 'getrawcode' : '._code.code:getrawcode', + 'patch_builtins' : '._code.code:patch_builtins', + 'unpatch_builtins' : '._code.code:unpatch_builtins', + '_AssertionError' : '._code.assertion:AssertionError', + '_reinterpret_old' : '._code.assertion:reinterpret_old', + '_reinterpret' : '._code.assertion:reinterpret', + }, + + # backports and additions of builtins + builtin = { + '__doc__' : '._builtin:__doc__', + 'enumerate' : '._builtin:enumerate', + 'reversed' : '._builtin:reversed', + 'sorted' : '._builtin:sorted', + 'set' : '._builtin:set', + 'frozenset' : '._builtin:frozenset', + 'BaseException' : '._builtin:BaseException', + 'GeneratorExit' : '._builtin:GeneratorExit', + 'print_' : '._builtin:print_', + '_reraise' : '._builtin:_reraise', + '_tryimport' : '._builtin:_tryimport', + 'exec_' : '._builtin:exec_', + '_basestring' : '._builtin:_basestring', + '_totext' : '._builtin:_totext', + '_isbytes' : '._builtin:_isbytes', + '_istext' : '._builtin:_istext', + '_getimself' : '._builtin:_getimself', + '_getfuncdict' : '._builtin:_getfuncdict', + '_getcode' : '._builtin:_getcode', + 'builtins' : '._builtin:builtins', + 'execfile' : '._builtin:execfile', + 'callable' : '._builtin:callable', + }, + + # input-output helping + io = { + '__doc__' : '._io:__doc__', + 'dupfile' : '._io.capture:dupfile', + 'TextIO' : '._io.capture:TextIO', + 'BytesIO' : '._io.capture:BytesIO', + 'FDCapture' : '._io.capture:FDCapture', + 'StdCapture' : '._io.capture:StdCapture', + 'StdCaptureFD' : '._io.capture:StdCaptureFD', + 'TerminalWriter' : '._io.terminalwriter:TerminalWriter', + 'ansi_print' : '._io.terminalwriter:ansi_print', + 'get_terminal_width' : '._io.terminalwriter:get_terminal_width', + 'saferepr' : '._io.saferepr:saferepr', + }, + + # small and mean xml/html generation + xml = { + '__doc__' : '._xmlgen:__doc__', + 'html' : '._xmlgen:html', + 'Tag' : '._xmlgen:Tag', + 'raw' : '._xmlgen:raw', + 'Namespace' : '._xmlgen:Namespace', + 'escape' : '._xmlgen:escape', + }, + + log = { + # logging API ('producers' and 'consumers' connected via keywords) + '__doc__' : '._log:__doc__', + '_apiwarn' : '._log.warning:_apiwarn', + 'Producer' : '._log.log:Producer', + 'setconsumer' : '._log.log:setconsumer', + '_setstate' : '._log.log:setstate', + '_getstate' : '._log.log:getstate', + 'Path' : '._log.log:Path', + 'STDOUT' : '._log.log:STDOUT', + 'STDERR' : '._log.log:STDERR', + 'Syslog' : '._log.log:Syslog', + }, + + # compatibility modules (deprecated) + compat = { + '__doc__' : '._compat:__doc__', + 'doctest' : '._compat.dep_doctest:doctest', + 'optparse' : '._compat.dep_optparse:optparse', + 'textwrap' : '._compat.dep_textwrap:textwrap', + 'subprocess' : '._compat.dep_subprocess:subprocess', + }, +)) Added: pypy/branch/py12/py/__init__.py.orig ============================================================================== --- (empty file) +++ pypy/branch/py12/py/__init__.py.orig Fri Apr 30 17:07:52 2010 @@ -0,0 +1,174 @@ +""" +py.test and pylib: rapid testing and development utils + +this module uses apipkg.py for lazy-loading sub modules +and classes. The initpkg-dictionary below specifies +name->value mappings where value can be another namespace +dictionary or an import path. + +(c) Holger Krekel and others, 2004-2010 +""" +__version__ = version = "1.2.2" + +import py.apipkg + +py.apipkg.initpkg(__name__, dict( + # access to all standard lib modules + std = '._std:std', + # access to all posix errno's as classes + error = '._error:error', + + _pydir = '.__metainfo:pydir', + version = 'py:__version__', # backward compatibility + + cmdline = { + 'pytest': '._cmdline.pytest:main', + 'pylookup': '._cmdline.pylookup:main', + 'pycountloc': '._cmdline.pycountlog:main', + 'pytest': '._test.cmdline:main', + 'pylookup': '._cmdline.pylookup:main', + 'pycountloc': '._cmdline.pycountloc:main', + 'pycleanup': '._cmdline.pycleanup:main', + 'pywhich' : '._cmdline.pywhich:main', + 'pysvnwcrevert' : '._cmdline.pysvnwcrevert:main', + 'pyconvert_unittest' : '._cmdline.pyconvert_unittest:main', + }, + + test = { + # helpers for use from test functions or collectors + '__onfirstaccess__' : '._test.config:onpytestaccess', + '__doc__' : '._test:__doc__', + # configuration/initialization related test api + 'config' : '._test.config:config_per_process', + 'ensuretemp' : '._test.config:ensuretemp', + 'collect': { + 'Collector' : '._test.collect:Collector', + 'Directory' : '._test.collect:Directory', + 'File' : '._test.collect:File', + 'Item' : '._test.collect:Item', + 'Module' : '._test.pycollect:Module', + 'Class' : '._test.pycollect:Class', + 'Instance' : '._test.pycollect:Instance', + 'Generator' : '._test.pycollect:Generator', + 'Function' : '._test.pycollect:Function', + '_fillfuncargs' : '._test.funcargs:fillfuncargs', + }, + 'cmdline': { + 'main' : '._test.cmdline:main', # backward compat + }, + }, + + # hook into the top-level standard library + process = { + '__doc__' : '._process:__doc__', + 'cmdexec' : '._process.cmdexec:cmdexec', + 'kill' : '._process.killproc:kill', + 'ForkedFunc' : '._process.forkedfunc:ForkedFunc', + }, + + path = { + '__doc__' : '._path:__doc__', + 'svnwc' : '._path.svnwc:SvnWCCommandPath', + 'svnurl' : '._path.svnurl:SvnCommandPath', + 'local' : '._path.local:LocalPath', + 'SvnAuth' : '._path.svnwc:SvnAuth', + }, + + # some nice slightly magic APIs + magic = { + 'invoke' : '._code.oldmagic:invoke', + 'revoke' : '._code.oldmagic:revoke', + 'patch' : '._code.oldmagic:patch', + 'revert' : '._code.oldmagic:revert', + 'autopath' : '._path.local:autopath', + 'AssertionError' : '._code.oldmagic2:AssertionError', + }, + + # python inspection/code-generation API + code = { + '__doc__' : '._code:__doc__', + 'compile' : '._code.source:compile_', + 'Source' : '._code.source:Source', + 'Code' : '._code.code:Code', + 'Frame' : '._code.code:Frame', + 'ExceptionInfo' : '._code.code:ExceptionInfo', + 'Traceback' : '._code.code:Traceback', + 'getfslineno' : '._code.source:getfslineno', + 'getrawcode' : '._code.code:getrawcode', + 'patch_builtins' : '._code.code:patch_builtins', + 'unpatch_builtins' : '._code.code:unpatch_builtins', + '_AssertionError' : '._code.assertion:AssertionError', + '_reinterpret_old' : '._code.assertion:reinterpret_old', + }, + + # backports and additions of builtins + builtin = { + '__doc__' : '._builtin:__doc__', + 'enumerate' : '._builtin:enumerate', + 'reversed' : '._builtin:reversed', + 'sorted' : '._builtin:sorted', + 'set' : '._builtin:set', + 'frozenset' : '._builtin:frozenset', + 'BaseException' : '._builtin:BaseException', + 'GeneratorExit' : '._builtin:GeneratorExit', + 'print_' : '._builtin:print_', + '_reraise' : '._builtin:_reraise', + '_tryimport' : '._builtin:_tryimport', + 'exec_' : '._builtin:exec_', + '_basestring' : '._builtin:_basestring', + '_totext' : '._builtin:_totext', + '_isbytes' : '._builtin:_isbytes', + '_istext' : '._builtin:_istext', + '_getimself' : '._builtin:_getimself', + '_getfuncdict' : '._builtin:_getfuncdict', + '_getcode' : '._builtin:_getcode', + 'builtins' : '._builtin:builtins', + 'execfile' : '._builtin:execfile', + 'callable' : '._builtin:callable', + }, + + # input-output helping + io = { + '__doc__' : '._io:__doc__', + 'dupfile' : '._io.capture:dupfile', + 'TextIO' : '._io.capture:TextIO', + 'BytesIO' : '._io.capture:BytesIO', + 'FDCapture' : '._io.capture:FDCapture', + 'StdCapture' : '._io.capture:StdCapture', + 'StdCaptureFD' : '._io.capture:StdCaptureFD', + 'TerminalWriter' : '._io.terminalwriter:TerminalWriter', + }, + + # small and mean xml/html generation + xml = { + '__doc__' : '._xmlgen:__doc__', + 'html' : '._xmlgen:html', + 'Tag' : '._xmlgen:Tag', + 'raw' : '._xmlgen:raw', + 'Namespace' : '._xmlgen:Namespace', + 'escape' : '._xmlgen:escape', + }, + + log = { + # logging API ('producers' and 'consumers' connected via keywords) + '__doc__' : '._log:__doc__', + '_apiwarn' : '._log.warning:_apiwarn', + 'Producer' : '._log.log:Producer', + 'setconsumer' : '._log.log:setconsumer', + '_setstate' : '._log.log:setstate', + '_getstate' : '._log.log:getstate', + 'Path' : '._log.log:Path', + 'STDOUT' : '._log.log:STDOUT', + 'STDERR' : '._log.log:STDERR', + 'Syslog' : '._log.log:Syslog', + }, + + # compatibility modules (deprecated) + compat = { + '__doc__' : '._compat:__doc__', + 'doctest' : '._compat.dep_doctest:doctest', + 'optparse' : '._compat.dep_optparse:optparse', + 'textwrap' : '._compat.dep_textwrap:textwrap', + 'subprocess' : '._compat.dep_subprocess:subprocess', + }, +)) Added: pypy/branch/py12/py/__metainfo.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/__metainfo.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,2 @@ +import py +pydir = py.path.local(py.__file__).dirpath() Added: pypy/branch/py12/py/_builtin.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_builtin.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,209 @@ +import sys + +try: + reversed = reversed +except NameError: + def reversed(sequence): + """reversed(sequence) -> reverse iterator over values of the sequence + + Return a reverse iterator + """ + if hasattr(sequence, '__reversed__'): + return sequence.__reversed__() + if not hasattr(sequence, '__getitem__'): + raise TypeError("argument to reversed() must be a sequence") + return reversed_iterator(sequence) + + class reversed_iterator(object): + + def __init__(self, seq): + self.seq = seq + self.remaining = len(seq) + + def __iter__(self): + return self + + def next(self): + i = self.remaining + if i > 0: + i -= 1 + item = self.seq[i] + self.remaining = i + return item + raise StopIteration + + def __length_hint__(self): + return self.remaining + +try: + sorted = sorted +except NameError: + builtin_cmp = cmp # need to use cmp as keyword arg + + def sorted(iterable, cmp=None, key=None, reverse=0): + use_cmp = None + if key is not None: + if cmp is None: + def use_cmp(x, y): + return builtin_cmp(x[0], y[0]) + else: + def use_cmp(x, y): + return cmp(x[0], y[0]) + l = [(key(element), element) for element in iterable] + else: + if cmp is not None: + use_cmp = cmp + l = list(iterable) + if use_cmp is not None: + l.sort(use_cmp) + else: + l.sort() + if reverse: + l.reverse() + if key is not None: + return [element for (_, element) in l] + return l + +try: + set, frozenset = set, frozenset +except NameError: + from sets import set, frozenset + +# pass through +enumerate = enumerate + +try: + BaseException = BaseException +except NameError: + BaseException = Exception + +try: + GeneratorExit = GeneratorExit +except NameError: + class GeneratorExit(Exception): + """ This exception is never raised, it is there to make it possible to + write code compatible with CPython 2.5 even in lower CPython + versions.""" + pass + GeneratorExit.__module__ = 'exceptions' + +if sys.version_info >= (3, 0): + exec ("print_ = print ; exec_=exec") + import builtins + + # some backward compatibility helpers + _basestring = str + def _totext(obj, encoding=None): + if isinstance(obj, bytes): + obj = obj.decode(encoding) + elif not isinstance(obj, str): + obj = str(obj) + return obj + + def _isbytes(x): + return isinstance(x, bytes) + def _istext(x): + return isinstance(x, str) + + def _getimself(function): + return getattr(function, '__self__', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def _getcode(function): + return getattr(function, "__code__", None) + + def execfile(fn, globs=None, locs=None): + if globs is None: + back = sys._getframe(1) + globs = back.f_globals + locs = back.f_locals + del back + elif locs is None: + locs = globs + fp = open(fn, "rb") + try: + source = fp.read() + finally: + fp.close() + co = compile(source, fn, "exec", dont_inherit=True) + exec_(co, globs, locs) + + def callable(obj): + return hasattr(obj, "__call__") + +else: + import __builtin__ as builtins + _totext = unicode + _basestring = basestring + execfile = execfile + callable = callable + def _isbytes(x): + return isinstance(x, str) + def _istext(x): + return isinstance(x, unicode) + + def _getimself(function): + return getattr(function, 'im_self', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def _getcode(function): + return getattr(function, "func_code", None) + + def print_(*args, **kwargs): + """ minimal backport of py3k print statement. """ + sep = ' ' + if 'sep' in kwargs: + sep = kwargs.pop('sep') + end = '\n' + if 'end' in kwargs: + end = kwargs.pop('end') + file = 'file' in kwargs and kwargs.pop('file') or sys.stdout + if kwargs: + args = ", ".join([str(x) for x in kwargs]) + raise TypeError("invalid keyword arguments: %s" % args) + at_start = True + for x in args: + if not at_start: + file.write(sep) + file.write(str(x)) + at_start = False + file.write(end) + + def exec_(obj, globals=None, locals=None): + """ minimal backport of py3k exec statement. """ + if globals is None: + frame = sys._getframe(1) + globals = frame.f_globals + if locals is None: + locals = frame.f_locals + elif locals is None: + locals = globals + exec2(obj, globals, locals) + +if sys.version_info >= (3,0): + exec (""" +def _reraise(cls, val, tb): + assert hasattr(val, '__traceback__') + raise val +""") +else: + exec (""" +def _reraise(cls, val, tb): + raise cls, val, tb +def exec2(obj, globals, locals): + exec obj in globals, locals +""") + +def _tryimport(*names): + """ return the first successfully imported module. """ + assert names + for name in names: + try: + return __import__(name, None, None, '__doc__') + except ImportError: + excinfo = sys.exc_info() + _reraise(*excinfo) Added: pypy/branch/py12/py/_cmdline/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +# Added: pypy/branch/py12/py/_cmdline/pycleanup.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pycleanup.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +"""\ +py.cleanup [PATH] ... + +Delete typical python development related files recursively under the specified PATH (which defaults to the current working directory). Don't follow links and don't recurse into directories with a dot. Optionally remove setup.py related files and empty +directories. + +""" +import py +import sys, subprocess + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + parser.add_option("-e", metavar="ENDING", + dest="endings", default=[".pyc", "$py.class"], action="append", + help=("(multi) recursively remove files with the given ending." + " '.pyc' and '$py.class' are in the default list.")) + parser.add_option("-d", action="store_true", dest="removedir", + help="remove empty directories.") + parser.add_option("-s", action="store_true", dest="setup", + help="remove 'build' and 'dist' directories next to setup.py files") + parser.add_option("-a", action="store_true", dest="all", + help="synonym for '-S -d -e pip-log.txt'") + parser.add_option("-n", "--dryrun", dest="dryrun", default=False, + action="store_true", + help="don't actually delete but display would-be-removed filenames.") + (options, args) = parser.parse_args() + + Cleanup(options, args).main() + +class Cleanup: + def __init__(self, options, args): + if not args: + args = ["."] + self.options = options + self.args = [py.path.local(x) for x in args] + if options.all: + options.setup = True + options.removedir = True + options.endings.append("pip-log.txt") + + def main(self): + if self.options.setup: + for arg in self.args: + self.setupclean(arg) + + for path in self.args: + py.builtin.print_("cleaning path", path, + "of extensions", self.options.endings) + for x in path.visit(self.shouldremove, self.recursedir): + self.remove(x) + if self.options.removedir: + for x in path.visit(lambda x: x.check(dir=1), self.recursedir): + if not x.listdir(): + self.remove(x) + + def shouldremove(self, p): + for ending in self.options.endings: + if p.basename.endswith(ending): + return True + + def recursedir(self, path): + return path.check(dotfile=0, link=0) + + def remove(self, path): + if not path.check(): + return + if self.options.dryrun: + py.builtin.print_("would remove", path) + else: + py.builtin.print_("removing", path) + path.remove() + + def XXXcallsetup(self, setup, *args): + old = setup.dirpath().chdir() + try: + subprocess.call([sys.executable, str(setup)] + list(args)) + finally: + old.chdir() + + def setupclean(self, path): + for x in path.visit("setup.py", self.recursedir): + basepath = x.dirpath() + self.remove(basepath / "build") + self.remove(basepath / "dist") Added: pypy/branch/py12/py/_cmdline/pyconvert_unittest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pyconvert_unittest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,253 @@ +import re +import sys + +try: + import parser +except ImportError: + parser = None + +d={} +# d is the dictionary of unittest changes, keyed to the old name +# used by unittest. +# d[old][0] is the new replacement function. +# d[old][1] is the operator you will substitute, or '' if there is none. +# d[old][2] is the possible number of arguments to the unittest +# function. + +# Old Unittest Name new name operator # of args +d['assertRaises'] = ('raises', '', ['Any']) +d['fail'] = ('raise AssertionError', '', [0,1]) +d['assert_'] = ('assert', '', [1,2]) +d['failIf'] = ('assert not', '', [1,2]) +d['assertEqual'] = ('assert', ' ==', [2,3]) +d['failIfEqual'] = ('assert not', ' ==', [2,3]) +d['assertIn'] = ('assert', ' in', [2,3]) +d['assertNotIn'] = ('assert', ' not in', [2,3]) +d['assertNotEqual'] = ('assert', ' !=', [2,3]) +d['failUnlessEqual'] = ('assert', ' ==', [2,3]) +d['assertAlmostEqual'] = ('assert round', ' ==', [2,3,4]) +d['failIfAlmostEqual'] = ('assert not round', ' ==', [2,3,4]) +d['assertNotAlmostEqual'] = ('assert round', ' !=', [2,3,4]) +d['failUnlessAlmostEquals'] = ('assert round', ' ==', [2,3,4]) + +# the list of synonyms +d['failUnlessRaises'] = d['assertRaises'] +d['failUnless'] = d['assert_'] +d['assertEquals'] = d['assertEqual'] +d['assertNotEquals'] = d['assertNotEqual'] +d['assertAlmostEquals'] = d['assertAlmostEqual'] +d['assertNotAlmostEquals'] = d['assertNotAlmostEqual'] + +# set up the regular expressions we will need +leading_spaces = re.compile(r'^(\s*)') # this never fails + +pat = '' +for k in d.keys(): # this complicated pattern to match all unittests + pat += '|' + r'^(\s*)' + 'self.' + k + r'\(' # \tself.whatever( + +old_names = re.compile(pat[1:]) +linesep='\n' # nobody will really try to convert files not read + # in text mode, will they? + + +def blocksplitter(fp): + '''split a file into blocks that are headed by functions to rename''' + + blocklist = [] + blockstring = '' + + for line in fp: + interesting = old_names.match(line) + if interesting : + if blockstring: + blocklist.append(blockstring) + blockstring = line # reset the block + else: + blockstring += line + + blocklist.append(blockstring) + return blocklist + +def rewrite_utest(block): + '''rewrite every block to use the new utest functions''' + + '''returns the rewritten unittest, unless it ran into problems, + in which case it just returns the block unchanged. + ''' + utest = old_names.match(block) + + if not utest: + return block + + old = utest.group(0).lstrip()[5:-1] # the name we want to replace + new = d[old][0] # the name of the replacement function + op = d[old][1] # the operator you will use , or '' if there is none. + possible_args = d[old][2] # a list of the number of arguments the + # unittest function could possibly take. + + if possible_args == ['Any']: # just rename assertRaises & friends + return re.sub('self.'+old, new, block) + + message_pos = possible_args[-1] + # the remaining unittests can have an optional message to print + # when they fail. It is always the last argument to the function. + + try: + indent, argl, trailer = decompose_unittest(old, block) + + except SyntaxError: # but we couldn't parse it! + return block + + argnum = len(argl) + if argnum not in possible_args: + # sanity check - this one isn't real either + return block + + elif argnum == message_pos: + message = argl[-1] + argl = argl[:-1] + else: + message = None + + if argnum is 0 or (argnum is 1 and argnum is message_pos): #unittest fail() + string = '' + if message: + message = ' ' + message + + elif message_pos is 4: # assertAlmostEqual & friends + try: + pos = argl[2].lstrip() + except IndexError: + pos = '7' # default if none is specified + string = '(%s -%s, %s)%s 0' % (argl[0], argl[1], pos, op ) + + else: # assert_, assertEquals and all the rest + string = ' ' + op.join(argl) + + if message: + string = string + ',' + message + + return indent + new + string + trailer + +def decompose_unittest(old, block): + '''decompose the block into its component parts''' + + ''' returns indent, arglist, trailer + indent -- the indentation + arglist -- the arguments to the unittest function + trailer -- any extra junk after the closing paren, such as #commment + ''' + + indent = re.match(r'(\s*)', block).group() + pat = re.search('self.' + old + r'\(', block) + + args, trailer = get_expr(block[pat.end():], ')') + arglist = break_args(args, []) + + if arglist == ['']: # there weren't any + return indent, [], trailer + + for i in range(len(arglist)): + try: + parser.expr(arglist[i].lstrip('\t ')) + except SyntaxError: + if i == 0: + arglist[i] = '(' + arglist[i] + ')' + else: + arglist[i] = ' (' + arglist[i] + ')' + + return indent, arglist, trailer + +def break_args(args, arglist): + '''recursively break a string into a list of arguments''' + try: + first, rest = get_expr(args, ',') + if not rest: + return arglist + [first] + else: + return [first] + break_args(rest, arglist) + except SyntaxError: + return arglist + [args] + +def get_expr(s, char): + '''split a string into an expression, and the rest of the string''' + + pos=[] + for i in range(len(s)): + if s[i] == char: + pos.append(i) + if pos == []: + raise SyntaxError # we didn't find the expected char. Ick. + + for p in pos: + # make the python parser do the hard work of deciding which comma + # splits the string into two expressions + try: + parser.expr('(' + s[:p] + ')') + return s[:p], s[p+1:] + except SyntaxError: # It's not an expression yet + pass + raise SyntaxError # We never found anything that worked. + + +def main(): + import sys + import py + + usage = "usage: %prog [-s [filename ...] | [-i | -c filename ...]]" + optparser = py.std.optparse.OptionParser(usage) + + def select_output (option, opt, value, optparser, **kw): + if hasattr(optparser, 'output'): + optparser.error( + 'Cannot combine -s -i and -c options. Use one only.') + else: + optparser.output = kw['output'] + + optparser.add_option("-s", "--stdout", action="callback", + callback=select_output, + callback_kwargs={'output':'stdout'}, + help="send your output to stdout") + + optparser.add_option("-i", "--inplace", action="callback", + callback=select_output, + callback_kwargs={'output':'inplace'}, + help="overwrite files in place") + + optparser.add_option("-c", "--copy", action="callback", + callback=select_output, + callback_kwargs={'output':'copy'}, + help="copy files ... fn.py --> fn_cp.py") + + options, args = optparser.parse_args() + + output = getattr(optparser, 'output', 'stdout') + + if output in ['inplace', 'copy'] and not args: + optparser.error( + '-i and -c option require at least one filename') + + if not args: + s = '' + for block in blocksplitter(sys.stdin): + s += rewrite_utest(block) + sys.stdout.write(s) + + else: + for infilename in args: # no error checking to see if we can open, etc. + infile = file(infilename) + s = '' + for block in blocksplitter(infile): + s += rewrite_utest(block) + if output == 'inplace': + outfile = file(infilename, 'w+') + elif output == 'copy': # yes, just go clobber any existing .cp + outfile = file (infilename[:-3]+ '_cp.py', 'w+') + else: + outfile = sys.stdout + + outfile.write(s) + + +if __name__ == '__main__': + main() Added: pypy/branch/py12/py/_cmdline/pycountloc.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pycountloc.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# hands on script to compute the non-empty Lines of Code +# for tests and non-test code + +"""\ +py.countloc [PATHS] + +Count (non-empty) lines of python code and number of python files recursively +starting from a list of paths given on the command line (starting from the +current working directory). Distinguish between test files and normal ones and +report them separately. +""" +import py + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + (options, args) = parser.parse_args() + countloc(args) + +def nodot(p): + return p.check(dotfile=0) + +class FileCounter(object): + def __init__(self): + self.file2numlines = {} + self.numlines = 0 + self.numfiles = 0 + + def addrecursive(self, directory, fil="*.py", rec=nodot): + for x in directory.visit(fil, rec): + self.addfile(x) + + def addfile(self, fn, emptylines=False): + if emptylines: + s = len(p.readlines()) + else: + s = 0 + for i in fn.readlines(): + if i.strip(): + s += 1 + self.file2numlines[fn] = s + self.numfiles += 1 + self.numlines += s + + def getnumlines(self, fil): + numlines = 0 + for path, value in self.file2numlines.items(): + if fil(path): + numlines += value + return numlines + + def getnumfiles(self, fil): + numfiles = 0 + for path in self.file2numlines: + if fil(path): + numfiles += 1 + return numfiles + +def get_loccount(locations=None): + if locations is None: + localtions = [py.path.local()] + counter = FileCounter() + for loc in locations: + counter.addrecursive(loc, '*.py', rec=nodot) + + def istestfile(p): + return p.check(fnmatch='test_*.py') + isnottestfile = lambda x: not istestfile(x) + + numfiles = counter.getnumfiles(isnottestfile) + numlines = counter.getnumlines(isnottestfile) + numtestfiles = counter.getnumfiles(istestfile) + numtestlines = counter.getnumlines(istestfile) + + return counter, numfiles, numlines, numtestfiles, numtestlines + +def countloc(paths=None): + if not paths: + paths = ['.'] + locations = [py.path.local(x) for x in paths] + (counter, numfiles, numlines, numtestfiles, + numtestlines) = get_loccount(locations) + + items = counter.file2numlines.items() + items.sort(lambda x,y: cmp(x[1], y[1])) + for x, y in items: + print("%3d %30s" % (y,x)) + + print("%30s %3d" %("number of testfiles", numtestfiles)) + print("%30s %3d" %("number of non-empty testlines", numtestlines)) + print("%30s %3d" %("number of files", numfiles)) + print("%30s %3d" %("number of non-empty lines", numlines)) + Added: pypy/branch/py12/py/_cmdline/pylookup.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pylookup.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +"""\ +py.lookup [search_directory] SEARCH_STRING [options] + +Looks recursively at Python files for a SEARCH_STRING, starting from the +present working directory. Prints the line, with the filename and line-number +prepended.""" + +import sys, os +import py +from py.io import ansi_print, get_terminal_width +import re + +def rec(p): + return p.check(dotfile=0) + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-i", "--ignore-case", action="store_true", dest="ignorecase", + help="ignore case distinctions") +parser.add_option("-C", "--context", action="store", type="int", dest="context", + default=0, help="How many lines of output to show") + +terminal_width = get_terminal_width() + +def find_indexes(search_line, string): + indexes = [] + before = 0 + while 1: + i = search_line.find(string, before) + if i == -1: + break + indexes.append(i) + before = i + len(string) + return indexes + +def main(): + (options, args) = parser.parse_args() + if len(args) == 2: + search_dir, string = args + search_dir = py.path.local(search_dir) + else: + search_dir = py.path.local() + string = args[0] + if options.ignorecase: + string = string.lower() + for x in search_dir.visit('*.py', rec): + # match filename directly + s = x.relto(search_dir) + if options.ignorecase: + s = s.lower() + if s.find(string) != -1: + sys.stdout.write("%s: filename matches %r" %(x, string) + "\n") + + try: + s = x.read() + except py.error.ENOENT: + pass # whatever, probably broken link (ie emacs lock) + searchs = s + if options.ignorecase: + searchs = s.lower() + if s.find(string) != -1: + lines = s.splitlines() + if options.ignorecase: + searchlines = s.lower().splitlines() + else: + searchlines = lines + for i, (line, searchline) in enumerate(zip(lines, searchlines)): + indexes = find_indexes(searchline, string) + if not indexes: + continue + if not options.context: + sys.stdout.write("%s:%d: " %(x.relto(search_dir), i+1)) + last_index = 0 + for index in indexes: + sys.stdout.write(line[last_index: index]) + ansi_print(line[index: index+len(string)], + file=sys.stdout, esc=31, newline=False) + last_index = index + len(string) + sys.stdout.write(line[last_index:] + "\n") + else: + context = (options.context)/2 + for count in range(max(0, i-context), min(len(lines) - 1, i+context+1)): + print("%s:%d: %s" %(x.relto(search_dir), count+1, lines[count].rstrip())) + print("-" * terminal_width) Added: pypy/branch/py12/py/_cmdline/pysvnwcrevert.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pysvnwcrevert.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,55 @@ +#! /usr/bin/env python +"""\ +py.svnwcrevert [options] WCPATH + +Running this script and then 'svn up' puts the working copy WCPATH in a state +as clean as a fresh check-out. + +WARNING: you'll loose all local changes, obviously! + +This script deletes all files that have been modified +or that svn doesn't explicitly know about, including svn:ignored files +(like .pyc files, hint hint). + +The goal of this script is to leave the working copy with some files and +directories possibly missing, but - most importantly - in a state where +the following 'svn up' won't just crash. +""" + +import sys, py + +def kill(p, root): + print('< %s' % (p.relto(root),)) + p.remove(rec=1) + +def svnwcrevert(path, root=None, precious=[]): + if root is None: + root = path + wcpath = py.path.svnwc(path) + try: + st = wcpath.status() + except ValueError: # typically, "bad char in wcpath" + kill(path, root) + return + for p in path.listdir(): + if p.basename == '.svn' or p.basename in precious: + continue + wcp = py.path.svnwc(p) + if wcp not in st.unchanged and wcp not in st.external: + kill(p, root) + elif p.check(dir=1): + svnwcrevert(p, root) + +# XXX add a functional test + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-p", "--precious", + action="append", dest="precious", default=[], + help="preserve files with this name") + +def main(): + opts, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + svnwcrevert(py.path.local(args[0]), precious=opts.precious) Added: pypy/branch/py12/py/_cmdline/pytest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pytest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,5 @@ +#!/usr/bin/env python +import py + +def main(args): + py.test.cmdline.main(args) Added: pypy/branch/py12/py/_cmdline/pywhich.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_cmdline/pywhich.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +"""\ +py.which [name] + +print the location of the given python module or package name +""" + +import sys + +def main(): + name = sys.argv[1] + try: + mod = __import__(name) + except ImportError: + sys.stderr.write("could not import: " + name + "\n") + else: + try: + location = mod.__file__ + except AttributeError: + sys.stderr.write("module (has no __file__): " + str(mod)) + else: + print(location) Added: pypy/branch/py12/py/_code/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +""" python inspection/code generation API """ Added: pypy/branch/py12/py/_code/_assertionnew.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/_assertionnew.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,337 @@ +""" +Like _assertion.py but using builtin AST. It should replace _assertionold.py +eventually. +""" + +import sys +import ast + +import py +from py._code.assertion import _format_explanation, BuiltinAssertionError + + +if sys.platform.startswith("java") and sys.version_info < (2, 5, 2): + # See http://bugs.jython.org/issue1497 + _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict", + "ListComp", "GeneratorExp", "Yield", "Compare", "Call", + "Repr", "Num", "Str", "Attribute", "Subscript", "Name", + "List", "Tuple") + _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign", + "AugAssign", "Print", "For", "While", "If", "With", "Raise", + "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom", + "Exec", "Global", "Expr", "Pass", "Break", "Continue") + _expr_nodes = set(getattr(ast, name) for name in _exprs) + _stmt_nodes = set(getattr(ast, name) for name in _stmts) + def _is_ast_expr(node): + return node.__class__ in _expr_nodes + def _is_ast_stmt(node): + return node.__class__ in _stmt_nodes +else: + def _is_ast_expr(node): + return isinstance(node, ast.expr) + def _is_ast_stmt(node): + return isinstance(node, ast.stmt) + + +class Failure(Exception): + """Error found while interpreting AST.""" + + def __init__(self, explanation=""): + self.cause = sys.exc_info() + self.explanation = explanation + + +def interpret(source, frame, should_fail=False): + mod = ast.parse(source) + visitor = DebugInterpreter(frame) + try: + visitor.visit(mod) + except Failure: + failure = sys.exc_info()[1] + return getfailure(failure) + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --no-assert)") + +def run(offending_line, frame=None): + if frame is None: + frame = py.code.Frame(sys._getframe(1)) + return interpret(offending_line, frame) + +def getfailure(failure): + explanation = _format_explanation(failure.explanation) + value = failure.cause[1] + if str(value): + lines = explanation.splitlines() + if not lines: + lines.append("") + lines[0] += " << %s" % (value,) + explanation = "\n".join(lines) + text = "%s: %s" % (failure.cause[0].__name__, explanation) + if text.startswith("AssertionError: assert "): + text = text[16:] + return text + + +operator_map = { + ast.BitOr : "|", + ast.BitXor : "^", + ast.BitAnd : "&", + ast.LShift : "<<", + ast.RShift : ">>", + ast.Add : "+", + ast.Sub : "-", + ast.Mult : "*", + ast.Div : "/", + ast.FloorDiv : "//", + ast.Mod : "%", + ast.Eq : "==", + ast.NotEq : "!=", + ast.Lt : "<", + ast.LtE : "<=", + ast.Gt : ">", + ast.GtE : ">=", + ast.Is : "is", + ast.IsNot : "is not", + ast.In : "in", + ast.NotIn : "not in" +} + +unary_map = { + ast.Not : "not %s", + ast.Invert : "~%s", + ast.USub : "-%s", + ast.UAdd : "+%s" +} + + +class DebugInterpreter(ast.NodeVisitor): + """Interpret AST nodes to gleam useful debugging information.""" + + def __init__(self, frame): + self.frame = frame + + def generic_visit(self, node): + # Fallback when we don't have a special implementation. + if _is_ast_expr(node): + mod = ast.Expression(node) + co = self._compile(mod) + try: + result = self.frame.eval(co) + except Exception: + raise Failure() + explanation = self.frame.repr(result) + return explanation, result + elif _is_ast_stmt(node): + mod = ast.Module([node]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co) + except Exception: + raise Failure() + return None, None + else: + raise AssertionError("can't handle %s" %(node,)) + + def _compile(self, source, mode="eval"): + return compile(source, "", mode) + + def visit_Expr(self, expr): + return self.visit(expr.value) + + def visit_Module(self, mod): + for stmt in mod.body: + self.visit(stmt) + + def visit_Name(self, name): + explanation, result = self.generic_visit(name) + # See if the name is local. + source = "%r in locals() is not globals()" % (name.id,) + co = self._compile(source) + try: + local = self.frame.eval(co) + except Exception: + # have to assume it isn't + local = False + if not local: + return name.id, result + return explanation, result + + def visit_Compare(self, comp): + left = comp.left + left_explanation, left_result = self.visit(left) + got_result = False + for op, next_op in zip(comp.ops, comp.comparators): + if got_result and not result: + break + next_explanation, next_result = self.visit(next_op) + op_symbol = operator_map[op.__class__] + explanation = "%s %s %s" % (left_explanation, op_symbol, + next_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=next_result) + except Exception: + raise Failure(explanation) + else: + got_result = True + left_explanation, left_result = next_explanation, next_result + return explanation, result + + def visit_BoolOp(self, boolop): + is_or = isinstance(boolop.op, ast.Or) + explanations = [] + for operand in boolop.values: + explanation, result = self.visit(operand) + explanations.append(explanation) + if result == is_or: + break + name = is_or and " or " or " and " + explanation = "(" + name.join(explanations) + ")" + return explanation, result + + def visit_UnaryOp(self, unary): + pattern = unary_map[unary.op.__class__] + operand_explanation, operand_result = self.visit(unary.operand) + explanation = pattern % (operand_explanation,) + co = self._compile(pattern % ("__exprinfo_expr",)) + try: + result = self.frame.eval(co, __exprinfo_expr=operand_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_BinOp(self, binop): + left_explanation, left_result = self.visit(binop.left) + right_explanation, right_result = self.visit(binop.right) + symbol = operator_map[binop.op.__class__] + explanation = "(%s %s %s)" % (left_explanation, symbol, + right_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=right_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_Call(self, call): + func_explanation, func = self.visit(call.func) + arg_explanations = [] + ns = {"__exprinfo_func" : func} + arguments = [] + for arg in call.args: + arg_explanation, arg_result = self.visit(arg) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + arguments.append(arg_name) + arg_explanations.append(arg_explanation) + for keyword in call.keywords: + arg_explanation, arg_result = self.visit(keyword.value) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + keyword_source = "%s=%%s" % (keyword.arg) + arguments.append(keyword_source % (arg_name,)) + arg_explanations.append(keyword_source % (arg_explanation,)) + if call.starargs: + arg_explanation, arg_result = self.visit(call.starargs) + arg_name = "__exprinfo_star" + ns[arg_name] = arg_result + arguments.append("*%s" % (arg_name,)) + arg_explanations.append("*%s" % (arg_explanation,)) + if call.kwargs: + arg_explanation, arg_result = self.visit(call.kwargs) + arg_name = "__exprinfo_kwds" + ns[arg_name] = arg_result + arguments.append("**%s" % (arg_name,)) + arg_explanations.append("**%s" % (arg_explanation,)) + args_explained = ", ".join(arg_explanations) + explanation = "%s(%s)" % (func_explanation, args_explained) + args = ", ".join(arguments) + source = "__exprinfo_func(%s)" % (args,) + co = self._compile(source) + try: + result = self.frame.eval(co, **ns) + except Exception: + raise Failure(explanation) + # Only show result explanation if it's not a builtin call or returns a + # bool. + if not isinstance(call.func, ast.Name) or \ + not self._is_builtin_name(call.func): + source = "isinstance(__exprinfo_value, bool)" + co = self._compile(source) + try: + is_bool = self.frame.eval(co, __exprinfo_value=result) + except Exception: + is_bool = False + if not is_bool: + pattern = "%s\n{%s = %s\n}" + rep = self.frame.repr(result) + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def _is_builtin_name(self, name): + pattern = "%r not in globals() and %r not in locals()" + source = pattern % (name.id, name.id) + co = self._compile(source) + try: + return self.frame.eval(co) + except Exception: + return False + + def visit_Attribute(self, attr): + if not isinstance(attr.ctx, ast.Load): + return self.generic_visit(attr) + source_explanation, source_result = self.visit(attr.value) + explanation = "%s.%s" % (source_explanation, attr.attr) + source = "__exprinfo_expr.%s" % (attr.attr,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + raise Failure(explanation) + # Check if the attr is from an instance. + source = "%r in getattr(__exprinfo_expr, '__dict__', {})" + source = source % (attr.attr,) + co = self._compile(source) + try: + from_instance = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + from_instance = True + if from_instance: + rep = self.frame.repr(result) + pattern = "%s\n{%s = %s\n}" + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def visit_Assert(self, assrt): + test_explanation, test_result = self.visit(assrt.test) + if test_explanation.startswith("False\n{False =") and \ + test_explanation.endswith("\n"): + test_explanation = test_explanation[15:-2] + explanation = "assert %s" % (test_explanation,) + if not test_result: + try: + raise BuiltinAssertionError + except Exception: + raise Failure(explanation) + return explanation, test_result + + def visit_Assign(self, assign): + value_explanation, value_result = self.visit(assign.value) + explanation = "... = %s" % (value_explanation,) + name = ast.Name("__exprinfo_expr", ast.Load(), assign.value.lineno, + assign.value.col_offset) + new_assign = ast.Assign(assign.targets, name, assign.lineno, + assign.col_offset) + mod = ast.Module([new_assign]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co, __exprinfo_expr=value_result) + except Exception: + raise Failure(explanation) + return explanation, value_result Added: pypy/branch/py12/py/_code/_assertionold.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/_assertionold.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,556 @@ +import py +import sys, inspect +from compiler import parse, ast, pycodegen +from py._code.assertion import BuiltinAssertionError, _format_explanation + +passthroughex = (KeyboardInterrupt, SystemExit, MemoryError) + +class Failure: + def __init__(self, node): + self.exc, self.value, self.tb = sys.exc_info() + self.node = node + +class View(object): + """View base class. + + If C is a subclass of View, then C(x) creates a proxy object around + the object x. The actual class of the proxy is not C in general, + but a *subclass* of C determined by the rules below. To avoid confusion + we call view class the class of the proxy (a subclass of C, so of View) + and object class the class of x. + + Attributes and methods not found in the proxy are automatically read on x. + Other operations like setting attributes are performed on the proxy, as + determined by its view class. The object x is available from the proxy + as its __obj__ attribute. + + The view class selection is determined by the __view__ tuples and the + optional __viewkey__ method. By default, the selected view class is the + most specific subclass of C whose __view__ mentions the class of x. + If no such subclass is found, the search proceeds with the parent + object classes. For example, C(True) will first look for a subclass + of C with __view__ = (..., bool, ...) and only if it doesn't find any + look for one with __view__ = (..., int, ...), and then ..., object,... + If everything fails the class C itself is considered to be the default. + + Alternatively, the view class selection can be driven by another aspect + of the object x, instead of the class of x, by overriding __viewkey__. + See last example at the end of this module. + """ + + _viewcache = {} + __view__ = () + + def __new__(rootclass, obj, *args, **kwds): + self = object.__new__(rootclass) + self.__obj__ = obj + self.__rootclass__ = rootclass + key = self.__viewkey__() + try: + self.__class__ = self._viewcache[key] + except KeyError: + self.__class__ = self._selectsubclass(key) + return self + + def __getattr__(self, attr): + # attributes not found in the normal hierarchy rooted on View + # are looked up in the object's real class + return getattr(self.__obj__, attr) + + def __viewkey__(self): + return self.__obj__.__class__ + + def __matchkey__(self, key, subclasses): + if inspect.isclass(key): + keys = inspect.getmro(key) + else: + keys = [key] + for key in keys: + result = [C for C in subclasses if key in C.__view__] + if result: + return result + return [] + + def _selectsubclass(self, key): + subclasses = list(enumsubclasses(self.__rootclass__)) + for C in subclasses: + if not isinstance(C.__view__, tuple): + C.__view__ = (C.__view__,) + choices = self.__matchkey__(key, subclasses) + if not choices: + return self.__rootclass__ + elif len(choices) == 1: + return choices[0] + else: + # combine the multiple choices + return type('?', tuple(choices), {}) + + def __repr__(self): + return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__) + + +def enumsubclasses(cls): + for subcls in cls.__subclasses__(): + for subsubclass in enumsubclasses(subcls): + yield subsubclass + yield cls + + +class Interpretable(View): + """A parse tree node with a few extra methods.""" + explanation = None + + def is_builtin(self, frame): + return False + + def eval(self, frame): + # fall-back for unknown expression nodes + try: + expr = ast.Expression(self.__obj__) + expr.filename = '' + self.__obj__.filename = '' + co = pycodegen.ExpressionCodeGenerator(expr).getCode() + result = frame.eval(co) + except passthroughex: + raise + except: + raise Failure(self) + self.result = result + self.explanation = self.explanation or frame.repr(self.result) + + def run(self, frame): + # fall-back for unknown statement nodes + try: + expr = ast.Module(None, ast.Stmt([self.__obj__])) + expr.filename = '' + co = pycodegen.ModuleCodeGenerator(expr).getCode() + frame.exec_(co) + except passthroughex: + raise + except: + raise Failure(self) + + def nice_explanation(self): + return _format_explanation(self.explanation) + + +class Name(Interpretable): + __view__ = ast.Name + + def is_local(self, frame): + co = compile('%r in locals() is not globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_global(self, frame): + co = compile('%r in globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_builtin(self, frame): + co = compile('%r not in locals() and %r not in globals()' % ( + self.name, self.name), '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + super(Name, self).eval(frame) + if not self.is_local(frame): + self.explanation = self.name + +class Compare(Interpretable): + __view__ = ast.Compare + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + for operation, expr2 in self.ops: + if hasattr(self, 'result'): + # shortcutting in chained expressions + if not frame.is_true(self.result): + break + expr2 = Interpretable(expr2) + expr2.eval(frame) + self.explanation = "%s %s %s" % ( + expr.explanation, operation, expr2.explanation) + co = compile("__exprinfo_left %s __exprinfo_right" % operation, + '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_left=expr.result, + __exprinfo_right=expr2.result) + except passthroughex: + raise + except: + raise Failure(self) + expr = expr2 + +class And(Interpretable): + __view__ = ast.And + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if not frame.is_true(expr.result): + break + self.explanation = '(' + ' and '.join(explanations) + ')' + +class Or(Interpretable): + __view__ = ast.Or + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if frame.is_true(expr.result): + break + self.explanation = '(' + ' or '.join(explanations) + ')' + + +# == Unary operations == +keepalive = [] +for astclass, astpattern in { + ast.Not : 'not __exprinfo_expr', + ast.Invert : '(~__exprinfo_expr)', + }.items(): + + class UnaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + expr = Interpretable(self.expr) + expr.eval(frame) + self.explanation = astpattern.replace('__exprinfo_expr', + expr.explanation) + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(UnaryArith) + +# == Binary operations == +for astclass, astpattern in { + ast.Add : '(__exprinfo_left + __exprinfo_right)', + ast.Sub : '(__exprinfo_left - __exprinfo_right)', + ast.Mul : '(__exprinfo_left * __exprinfo_right)', + ast.Div : '(__exprinfo_left / __exprinfo_right)', + ast.Mod : '(__exprinfo_left % __exprinfo_right)', + ast.Power : '(__exprinfo_left ** __exprinfo_right)', + }.items(): + + class BinaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + left = Interpretable(self.left) + left.eval(frame) + right = Interpretable(self.right) + right.eval(frame) + self.explanation = (astpattern + .replace('__exprinfo_left', left .explanation) + .replace('__exprinfo_right', right.explanation)) + try: + self.result = frame.eval(co, __exprinfo_left=left.result, + __exprinfo_right=right.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(BinaryArith) + + +class CallFunc(Interpretable): + __view__ = ast.CallFunc + + def is_bool(self, frame): + co = compile('isinstance(__exprinfo_value, bool)', '?', 'eval') + try: + return frame.is_true(frame.eval(co, __exprinfo_value=self.result)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + node = Interpretable(self.node) + node.eval(frame) + explanations = [] + vars = {'__exprinfo_fn': node.result} + source = '__exprinfo_fn(' + for a in self.args: + if isinstance(a, ast.Keyword): + keyword = a.name + a = a.expr + else: + keyword = None + a = Interpretable(a) + a.eval(frame) + argname = '__exprinfo_%d' % len(vars) + vars[argname] = a.result + if keyword is None: + source += argname + ',' + explanations.append(a.explanation) + else: + source += '%s=%s,' % (keyword, argname) + explanations.append('%s=%s' % (keyword, a.explanation)) + if self.star_args: + star_args = Interpretable(self.star_args) + star_args.eval(frame) + argname = '__exprinfo_star' + vars[argname] = star_args.result + source += '*' + argname + ',' + explanations.append('*' + star_args.explanation) + if self.dstar_args: + dstar_args = Interpretable(self.dstar_args) + dstar_args.eval(frame) + argname = '__exprinfo_kwds' + vars[argname] = dstar_args.result + source += '**' + argname + ',' + explanations.append('**' + dstar_args.explanation) + self.explanation = "%s(%s)" % ( + node.explanation, ', '.join(explanations)) + if source.endswith(','): + source = source[:-1] + source += ')' + co = compile(source, '?', 'eval') + try: + self.result = frame.eval(co, **vars) + except passthroughex: + raise + except: + raise Failure(self) + if not node.is_builtin(frame) or not self.is_bool(frame): + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +class Getattr(Interpretable): + __view__ = ast.Getattr + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + co = compile('__exprinfo_expr.%s' % self.attrname, '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + self.explanation = '%s.%s' % (expr.explanation, self.attrname) + # if the attribute comes from the instance, its value is interesting + co = compile('hasattr(__exprinfo_expr, "__dict__") and ' + '%r in __exprinfo_expr.__dict__' % self.attrname, + '?', 'eval') + try: + from_instance = frame.is_true( + frame.eval(co, __exprinfo_expr=expr.result)) + except passthroughex: + raise + except: + from_instance = True + if from_instance: + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +# == Re-interpretation of full statements == + +class Assert(Interpretable): + __view__ = ast.Assert + + def run(self, frame): + test = Interpretable(self.test) + test.eval(frame) + # simplify 'assert False where False = ...' + if (test.explanation.startswith('False\n{False = ') and + test.explanation.endswith('\n}')): + test.explanation = test.explanation[15:-2] + # print the result as 'assert ' + self.result = test.result + self.explanation = 'assert ' + test.explanation + if not frame.is_true(test.result): + try: + raise BuiltinAssertionError + except passthroughex: + raise + except: + raise Failure(self) + +class Assign(Interpretable): + __view__ = ast.Assign + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = '... = ' + expr.explanation + # fall-back-run the rest of the assignment + ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr')) + mod = ast.Module(None, ast.Stmt([ass])) + mod.filename = '' + co = pycodegen.ModuleCodeGenerator(mod).getCode() + try: + frame.exec_(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + +class Discard(Interpretable): + __view__ = ast.Discard + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = expr.explanation + +class Stmt(Interpretable): + __view__ = ast.Stmt + + def run(self, frame): + for stmt in self.nodes: + stmt = Interpretable(stmt) + stmt.run(frame) + + +def report_failure(e): + explanation = e.node.nice_explanation() + if explanation: + explanation = ", in: " + explanation + else: + explanation = "" + sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation)) + +def check(s, frame=None): + if frame is None: + frame = sys._getframe(1) + frame = py.code.Frame(frame) + expr = parse(s, 'eval') + assert isinstance(expr, ast.Expression) + node = Interpretable(expr.node) + try: + node.eval(frame) + except passthroughex: + raise + except Failure: + e = sys.exc_info()[1] + report_failure(e) + else: + if not frame.is_true(node.result): + sys.stderr.write("assertion failed: %s\n" % node.nice_explanation()) + + +########################################################### +# API / Entry points +# ######################################################### + +def interpret(source, frame, should_fail=False): + module = Interpretable(parse(source, 'exec').node) + #print "got module", module + if isinstance(frame, py.std.types.FrameType): + frame = py.code.Frame(frame) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + return getfailure(e) + except passthroughex: + raise + except: + import traceback + traceback.print_exc() + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --nomagic)") + else: + return None + +def getmsg(excinfo): + if isinstance(excinfo, tuple): + excinfo = py.code.ExceptionInfo(excinfo) + #frame, line = gettbline(tb) + #frame = py.code.Frame(frame) + #return interpret(line, frame) + + tb = excinfo.traceback[-1] + source = str(tb.statement).strip() + x = interpret(source, tb.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + return x + +def getfailure(e): + explanation = e.node.nice_explanation() + if str(e.value): + lines = explanation.split('\n') + lines[0] += " << %s" % (e.value,) + explanation = '\n'.join(lines) + text = "%s: %s" % (e.exc.__name__, explanation) + if text.startswith('AssertionError: assert '): + text = text[16:] + return text + +def run(s, frame=None): + if frame is None: + frame = sys._getframe(1) + frame = py.code.Frame(frame) + module = Interpretable(parse(s, 'exec').node) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + report_failure(e) + + +if __name__ == '__main__': + # example: + def f(): + return 5 + def g(): + return 3 + def h(x): + return 'never' + check("f() * g() == 5") + check("not f()") + check("not (f() and g() or 0)") + check("f() == g()") + i = 4 + check("i == f()") + check("len(f()) == 0") + check("isinstance(2+3+4, float)") + + run("x = i") + check("x == 5") + + run("assert not f(), 'oops'") + run("a, b, c = 1, 2") + run("a, b, c = f()") + + check("max([f(),g()]) == 4") + check("'hello'[g()] == 'h'") + run("'guk%d' % h(f())") Added: pypy/branch/py12/py/_code/assertion.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/assertion.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,77 @@ +import sys +import py + +BuiltinAssertionError = py.builtin.builtins.AssertionError + + +def _format_explanation(explanation): + # uck! See CallFunc for where \n{ and \n} escape sequences are used + raw_lines = (explanation or '').split('\n') + # escape newlines not followed by { and } + lines = [raw_lines[0]] + for l in raw_lines[1:]: + if l.startswith('{') or l.startswith('}'): + lines.append(l) + else: + lines[-1] += '\\n' + l + + result = lines[:1] + stack = [0] + stackcnt = [0] + for line in lines[1:]: + if line.startswith('{'): + if stackcnt[-1]: + s = 'and ' + else: + s = 'where ' + stack.append(len(result)) + stackcnt[-1] += 1 + stackcnt.append(0) + result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) + else: + assert line.startswith('}') + stack.pop() + stackcnt.pop() + result[stack[-1]] += line[1:] + assert len(stack) == 1 + return '\n'.join(result) + + +class AssertionError(BuiltinAssertionError): + + def __init__(self, *args): + BuiltinAssertionError.__init__(self, *args) + if args: + try: + self.msg = str(args[0]) + except (KeyboardInterrupt, SystemExit): + raise + except: + self.msg = "<[broken __repr__] %s at %0xd>" %( + args[0].__class__, id(args[0])) + else: + f = py.code.Frame(sys._getframe(1)) + try: + source = f.statement + source = str(source.deindent()).strip() + except py.error.ENOENT: + source = None + # this can also occur during reinterpretation, when the + # co_filename is set to "". + if source: + self.msg = reinterpret(source, f, should_fail=True) + if not self.args: + self.args = (self.msg,) + else: + self.msg = None + +if sys.version_info > (3, 0): + AssertionError.__module__ = "builtins" + reinterpret_old = "old reinterpretation not available for py3" +else: + from py._code._assertionold import interpret as reinterpret_old +if sys.version_info >= (2, 6) or (sys.platform.startswith("java")): + from py._code._assertionnew import interpret as reinterpret +else: + reinterpret = reinterpret_old + Added: pypy/branch/py12/py/_code/code.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/code.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,748 @@ +import py +import sys, os.path + +builtin_repr = repr + +reprlib = py.builtin._tryimport('repr', 'reprlib') + +class Code(object): + """ wrapper around Python code objects """ + def __init__(self, rawcode): + rawcode = py.code.getrawcode(rawcode) + self.raw = rawcode + try: + self.filename = rawcode.co_filename + self.firstlineno = rawcode.co_firstlineno - 1 + self.name = rawcode.co_name + except AttributeError: + raise TypeError("not a code object: %r" %(rawcode,)) + + def __eq__(self, other): + return self.raw == other.raw + + def __ne__(self, other): + return not self == other + + def new(self, rec=False, **kwargs): + """ return new code object with modified attributes. + if rec-cursive is true then dive into code + objects contained in co_consts. + """ + if sys.platform.startswith("java"): + # XXX jython does not support the below co_filename hack + return self.raw + names = [x for x in dir(self.raw) if x[:3] == 'co_'] + for name in kwargs: + if name not in names: + raise TypeError("unknown code attribute: %r" %(name, )) + if rec and hasattr(self.raw, 'co_consts'): # jython + newconstlist = [] + co = self.raw + cotype = type(co) + for c in co.co_consts: + if isinstance(c, cotype): + c = self.__class__(c).new(rec=True, **kwargs) + newconstlist.append(c) + return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) + for name in names: + if name not in kwargs: + kwargs[name] = getattr(self.raw, name) + arglist = [ + kwargs['co_argcount'], + kwargs['co_nlocals'], + kwargs.get('co_stacksize', 0), # jython + kwargs.get('co_flags', 0), # jython + kwargs.get('co_code', ''), # jython + kwargs.get('co_consts', ()), # jython + kwargs.get('co_names', []), # + kwargs['co_varnames'], + kwargs['co_filename'], + kwargs['co_name'], + kwargs['co_firstlineno'], + kwargs.get('co_lnotab', ''), #jython + kwargs.get('co_freevars', None), #jython + kwargs.get('co_cellvars', None), # jython + ] + if sys.version_info >= (3,0): + arglist.insert(1, kwargs['co_kwonlyargcount']) + return self.raw.__class__(*arglist) + else: + return py.std.new.code(*arglist) + + def path(self): + """ return a path object pointing to source code""" + fn = self.raw.co_filename + try: + return fn.__path__ + except AttributeError: + p = py.path.local(self.raw.co_filename) + if not p.check(): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p + + path = property(path, None, None, "path of this code object") + + def fullsource(self): + """ return a py.code.Source object for the full source file of the code + """ + from py._code import source + full, _ = source.findsource(self.raw) + return full + fullsource = property(fullsource, None, None, + "full source containing this code object") + + def source(self): + """ return a py.code.Source object for the code object's source only + """ + # return source only for that part of code + return py.code.Source(self.raw) + + def getargs(self): + """ return a tuple with the argument names for the code object + """ + # handfull shortcut for getting args + raw = self.raw + return raw.co_varnames[:raw.co_argcount] + +class Frame(object): + """Wrapper around a Python frame holding f_locals and f_globals + in which expressions can be evaluated.""" + + def __init__(self, frame): + self.code = py.code.Code(frame.f_code) + self.lineno = frame.f_lineno - 1 + self.f_globals = frame.f_globals + self.f_locals = frame.f_locals + self.raw = frame + + def statement(self): + if self.code.fullsource is None: + return py.code.Source("") + return self.code.fullsource.getstatement(self.lineno) + statement = property(statement, None, None, + "statement this frame is at") + + def eval(self, code, **vars): + """ evaluate 'code' in the frame + + 'vars' are optional additional local variables + + returns the result of the evaluation + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + return eval(code, self.f_globals, f_locals) + + def exec_(self, code, **vars): + """ exec 'code' in the frame + + 'vars' are optiona; additional local variables + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + py.builtin.exec_(code, self.f_globals, f_locals ) + + def repr(self, object): + """ return a 'safe' (non-recursive, one-line) string repr for 'object' + """ + return py.io.saferepr(object) + + def is_true(self, object): + return object + + def getargs(self): + """ return a list of tuples (name, value) for all arguments + """ + retval = [] + for arg in self.code.getargs(): + try: + retval.append((arg, self.f_locals[arg])) + except KeyError: + pass # this can occur when using Psyco + return retval + +class TracebackEntry(object): + """ a single entry in a traceback """ + + exprinfo = None + + def __init__(self, rawentry): + self._rawentry = rawentry + self.frame = py.code.Frame(rawentry.tb_frame) + # Ugh. 2.4 and 2.5 differs here when encountering + # multi-line statements. Not sure about the solution, but + # should be portable + self.lineno = rawentry.tb_lineno - 1 + self.relline = self.lineno - self.frame.code.firstlineno + + def __repr__(self): + return "" %(self.frame.code.path, self.lineno+1) + + def statement(self): + """ return a py.code.Source object for the current statement """ + source = self.frame.code.fullsource + return source.getstatement(self.lineno) + statement = property(statement, None, None, + "statement of this traceback entry.") + + def path(self): + return self.frame.code.path + path = property(path, None, None, "path to the full source code") + + def getlocals(self): + return self.frame.f_locals + locals = property(getlocals, None, None, "locals of underlaying frame") + + def reinterpret(self): + """Reinterpret the failing statement and returns a detailed information + about what operations are performed.""" + if self.exprinfo is None: + source = str(self.statement).strip() + x = py.code._reinterpret(source, self.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + self.exprinfo = x + return self.exprinfo + + def getfirstlinesource(self): + return self.frame.code.firstlineno + + def getsource(self): + """ return failing source code. """ + source = self.frame.code.fullsource + if source is None: + return None + start = self.getfirstlinesource() + end = self.lineno + try: + _, end = source.getstatementrange(end) + except IndexError: + end = self.lineno + 1 + # heuristic to stop displaying source on e.g. + # if something: # assume this causes a NameError + # # _this_ lines and the one + # below we don't want from entry.getsource() + for i in range(self.lineno, end): + if source[i].rstrip().endswith(':'): + end = i + 1 + break + return source[start:end] + source = property(getsource) + + def ishidden(self): + """ return True if the current frame has a var __tracebackhide__ + resolving to True + + mostly for internal use + """ + try: + return self.frame.eval("__tracebackhide__") + except (SystemExit, KeyboardInterrupt): + raise + except: + return False + + def __str__(self): + try: + fn = str(self.path) + except py.error.Error: + fn = '???' + name = self.frame.code.name + try: + line = str(self.statement).lstrip() + except KeyboardInterrupt: + raise + except: + line = "???" + return " File %r:%d in %s\n %s\n" %(fn, self.lineno+1, name, line) + + def name(self): + return self.frame.code.raw.co_name + name = property(name, None, None, "co_name of underlaying code") + +class Traceback(list): + """ Traceback objects encapsulate and offer higher level + access to Traceback entries. + """ + Entry = TracebackEntry + def __init__(self, tb): + """ initialize from given python traceback object. """ + if hasattr(tb, 'tb_next'): + def f(cur): + while cur is not None: + yield self.Entry(cur) + cur = cur.tb_next + list.__init__(self, f(tb)) + else: + list.__init__(self, tb) + + def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None): + """ return a Traceback instance wrapping part of this Traceback + + by provding any combination of path, lineno and firstlineno, the + first frame to start the to-be-returned traceback is determined + + this allows cutting the first part of a Traceback instance e.g. + for formatting reasons (removing some uninteresting bits that deal + with handling of the exception/traceback) + """ + for x in self: + code = x.frame.code + codepath = code.path + if ((path is None or codepath == path) and + (excludepath is None or not hasattr(codepath, 'relto') or + not codepath.relto(excludepath)) and + (lineno is None or x.lineno == lineno) and + (firstlineno is None or x.frame.code.firstlineno == firstlineno)): + return Traceback(x._rawentry) + return self + + def __getitem__(self, key): + val = super(Traceback, self).__getitem__(key) + if isinstance(key, type(slice(0))): + val = self.__class__(val) + return val + + def filter(self, fn=lambda x: not x.ishidden()): + """ return a Traceback instance with certain items removed + + fn is a function that gets a single argument, a TracebackItem + instance, and should return True when the item should be added + to the Traceback, False when not + + by default this removes all the TracebackItems which are hidden + (see ishidden() above) + """ + return Traceback(filter(fn, self)) + + def getcrashentry(self): + """ return last non-hidden traceback entry that lead + to the exception of a traceback. + """ + tb = self.filter() + if not tb: + tb = self + return tb[-1] + + def recursionindex(self): + """ return the index of the frame/TracebackItem where recursion + originates if appropriate, None if no recursion occurred + """ + cache = {} + for i, entry in enumerate(self): + key = entry.frame.code.path, entry.lineno + #print "checking for recursion at", key + l = cache.setdefault(key, []) + if l: + f = entry.frame + loc = f.f_locals + for otherloc in l: + if f.is_true(f.eval(co_equal, + __recursioncache_locals_1=loc, + __recursioncache_locals_2=otherloc)): + return i + l.append(entry.frame.f_locals) + return None + +co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2', + '?', 'eval') + +class ExceptionInfo(object): + """ wraps sys.exc_info() objects and offers + help for navigating the traceback. + """ + _striptext = '' + def __init__(self, tup=None, exprinfo=None): + # NB. all attributes are private! Subclasses or other + # ExceptionInfo-like classes may have different attributes. + if tup is None: + tup = sys.exc_info() + if exprinfo is None and isinstance(tup[1], py.code._AssertionError): + exprinfo = getattr(tup[1], 'msg', None) + if exprinfo is None: + exprinfo = str(tup[1]) + if exprinfo and exprinfo.startswith('assert '): + self._striptext = 'AssertionError: ' + self._excinfo = tup + self.type, self.value, tb = self._excinfo + self.typename = self.type.__name__ + self.traceback = py.code.Traceback(tb) + + def __repr__(self): + return "" % (self.typename, len(self.traceback)) + + def exconly(self, tryshort=False): + """ return the exception as a string + + when 'tryshort' resolves to True, and the exception is a + py.code._AssertionError, only the actual exception part of + the exception representation is returned (so 'AssertionError: ' is + removed from the beginning) + """ + lines = py.std.traceback.format_exception_only(self.type, self.value) + text = ''.join(lines) + text = text.rstrip() + if tryshort: + if text.startswith(self._striptext): + text = text[len(self._striptext):] + return text + + def errisinstance(self, exc): + """ return True if the exception is an instance of exc """ + return isinstance(self.value, exc) + + def _getreprcrash(self): + exconly = self.exconly(tryshort=True) + entry = self.traceback.getcrashentry() + path, lineno = entry.path, entry.lineno + reprcrash = ReprFileLocation(path, lineno+1, exconly) + return reprcrash + + def getrepr(self, showlocals=False, style="long", + abspath=False, tbfilter=True, funcargs=False): + """ return str()able representation of this exception info. + showlocals: show locals per traceback entry + style: long|short|no traceback style + tbfilter: hide entries (where __tracebackhide__ is true) + """ + fmt = FormattedExcinfo(showlocals=showlocals, style=style, + abspath=abspath, tbfilter=tbfilter, funcargs=funcargs) + return fmt.repr_excinfo(self) + + def __str__(self): + entry = self.traceback[-1] + loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly()) + return str(loc) + + def __unicode__(self): + entry = self.traceback[-1] + loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly()) + return unicode(loc) + + +class FormattedExcinfo(object): + """ presenting information about failing Functions and Generators. """ + # for traceback entries + flow_marker = ">" + fail_marker = "E" + + def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False): + self.showlocals = showlocals + self.style = style + self.tbfilter = tbfilter + self.funcargs = funcargs + self.abspath = abspath + + def _getindent(self, source): + # figure out indent for given source + try: + s = str(source.getstatement(len(source)-1)) + except KeyboardInterrupt: + raise + except: + try: + s = str(source[-1]) + except KeyboardInterrupt: + raise + except: + return 0 + return 4 + (len(s) - len(s.lstrip())) + + def _getentrysource(self, entry): + source = entry.getsource() + if source is not None: + source = source.deindent() + return source + + def _saferepr(self, obj): + return py.io.saferepr(obj) + + def repr_args(self, entry): + if self.funcargs: + args = [] + for argname, argvalue in entry.frame.getargs(): + args.append((argname, self._saferepr(argvalue))) + return ReprFuncArgs(args) + + def get_source(self, source, line_index=-1, excinfo=None): + """ return formatted and marked up source lines. """ + lines = [] + if source is None: + source = py.code.Source("???") + line_index = 0 + if line_index < 0: + line_index += len(source) + for i in range(len(source)): + if i == line_index: + prefix = self.flow_marker + " " + else: + prefix = " " + line = prefix + source[i] + lines.append(line) + if excinfo is not None: + indent = self._getindent(source) + lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) + return lines + + def get_exconly(self, excinfo, indent=4, markall=False): + lines = [] + indent = " " * indent + # get the real exception information out + exlines = excinfo.exconly(tryshort=True).split('\n') + failindent = self.fail_marker + indent[1:] + for line in exlines: + lines.append(failindent + line) + if not markall: + failindent = indent + return lines + + def repr_locals(self, locals): + if self.showlocals: + lines = [] + keys = list(locals) + keys.sort() + for name in keys: + value = locals[name] + if name == '__builtins__': + lines.append("__builtins__ = ") + else: + # This formatting could all be handled by the + # _repr() function, which is only reprlib.Repr in + # disguise, so is very configurable. + str_repr = self._saferepr(value) + #if len(str_repr) < 70 or not isinstance(value, + # (list, tuple, dict)): + lines.append("%-10s = %s" %(name, str_repr)) + #else: + # self._line("%-10s =\\" % (name,)) + # # XXX + # py.std.pprint.pprint(value, stream=self.excinfowriter) + return ReprLocals(lines) + + def repr_traceback_entry(self, entry, excinfo=None): + # excinfo is not None if this is the last tb entry + source = self._getentrysource(entry) + if source is None: + source = py.code.Source("???") + line_index = 0 + else: + # entry.getfirstlinesource() can be -1, should be 0 on jython + line_index = entry.lineno - max(entry.getfirstlinesource(), 0) + + lines = [] + if self.style == "long": + reprargs = self.repr_args(entry) + lines.extend(self.get_source(source, line_index, excinfo)) + message = excinfo and excinfo.typename or "" + path = self._makepath(entry.path) + filelocrepr = ReprFileLocation(path, entry.lineno+1, message) + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, filelocrepr) + else: + if self.style == "short": + line = source[line_index].lstrip() + basename = os.path.basename(entry.frame.code.filename) + lines.append(' File "%s", line %d, in %s' % ( + basename, entry.lineno+1, entry.name)) + lines.append(" " + line) + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None) + + def _makepath(self, path): + if not self.abspath: + np = py.path.local().bestrelpath(path) + if len(np) < len(str(path)): + path = np + return path + + def repr_traceback(self, excinfo): + traceback = excinfo.traceback + if self.tbfilter: + traceback = traceback.filter() + recursionindex = None + if excinfo.errisinstance(RuntimeError): + recursionindex = traceback.recursionindex() + last = traceback[-1] + entries = [] + extraline = None + for index, entry in enumerate(traceback): + einfo = (last == entry) and excinfo or None + reprentry = self.repr_traceback_entry(entry, einfo) + entries.append(reprentry) + if index == recursionindex: + extraline = "!!! Recursion detected (same locals & position)" + break + return ReprTraceback(entries, extraline, style=self.style) + + def repr_excinfo(self, excinfo): + reprtraceback = self.repr_traceback(excinfo) + reprcrash = excinfo._getreprcrash() + return ReprExceptionInfo(reprtraceback, reprcrash) + +class TerminalRepr: + def __str__(self): + s = self.__unicode__() + if sys.version_info[0] < 3: + s = s.encode('utf-8') + return s + + def __unicode__(self): + l = [] + tw = py.io.TerminalWriter(l.append) + self.toterminal(tw) + l = map(unicode_or_repr, l) + return "".join(l).strip() + + def __repr__(self): + return "<%s instance at %0x>" %(self.__class__, id(self)) + +def unicode_or_repr(obj): + try: + return py.builtin._totext(obj) + except KeyboardInterrupt: + raise + except Exception: + return "" % py.io.saferepr(obj) + +class ReprExceptionInfo(TerminalRepr): + def __init__(self, reprtraceback, reprcrash): + self.reprtraceback = reprtraceback + self.reprcrash = reprcrash + self.sections = [] + + def addsection(self, name, content, sep="-"): + self.sections.append((name, content, sep)) + + def toterminal(self, tw): + self.reprtraceback.toterminal(tw) + for name, content, sep in self.sections: + tw.sep(sep, name) + tw.line(content) + +class ReprTraceback(TerminalRepr): + entrysep = "_ " + + def __init__(self, reprentries, extraline, style): + self.reprentries = reprentries + self.extraline = extraline + self.style = style + + def toterminal(self, tw): + sepok = False + for entry in self.reprentries: + if self.style == "long": + if sepok: + tw.sep(self.entrysep) + tw.line("") + sepok = True + entry.toterminal(tw) + if self.extraline: + tw.line(self.extraline) + +class ReprEntry(TerminalRepr): + localssep = "_ " + + def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr): + self.lines = lines + self.reprfuncargs = reprfuncargs + self.reprlocals = reprlocals + self.reprfileloc = filelocrepr + + def toterminal(self, tw): + if self.reprfuncargs: + self.reprfuncargs.toterminal(tw) + for line in self.lines: + red = line.startswith("E ") + tw.line(line, bold=True, red=red) + if self.reprlocals: + #tw.sep(self.localssep, "Locals") + tw.line("") + self.reprlocals.toterminal(tw) + if self.reprfileloc: + tw.line("") + self.reprfileloc.toterminal(tw) + + def __str__(self): + return "%s\n%s\n%s" % ("\n".join(self.lines), + self.reprlocals, + self.reprfileloc) + +class ReprFileLocation(TerminalRepr): + def __init__(self, path, lineno, message): + self.path = str(path) + self.lineno = lineno + self.message = message + + def toterminal(self, tw): + # filename and lineno output for each entry, + # using an output format that most editors unterstand + msg = self.message + i = msg.find("\n") + if i != -1: + msg = msg[:i] + tw.line("%s:%s: %s" %(self.path, self.lineno, msg)) + +class ReprLocals(TerminalRepr): + def __init__(self, lines): + self.lines = lines + + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + +class ReprFuncArgs(TerminalRepr): + def __init__(self, args): + self.args = args + + def toterminal(self, tw): + if self.args: + linesofar = "" + for name, value in self.args: + ns = "%s = %s" %(name, value) + if len(ns) + len(linesofar) + 2 > tw.fullwidth: + if linesofar: + tw.line(linesofar) + linesofar = ns + else: + if linesofar: + linesofar += ", " + ns + else: + linesofar = ns + if linesofar: + tw.line(linesofar) + tw.line("") + + + +oldbuiltins = {} + +def patch_builtins(assertion=True, compile=True): + """ put compile and AssertionError builtins to Python's builtins. """ + if assertion: + from py._code import assertion + l = oldbuiltins.setdefault('AssertionError', []) + l.append(py.builtin.builtins.AssertionError) + py.builtin.builtins.AssertionError = assertion.AssertionError + if compile: + l = oldbuiltins.setdefault('compile', []) + l.append(py.builtin.builtins.compile) + py.builtin.builtins.compile = py.code.compile + +def unpatch_builtins(assertion=True, compile=True): + """ remove compile and AssertionError builtins from Python builtins. """ + if assertion: + py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop() + if compile: + py.builtin.builtins.compile = oldbuiltins['compile'].pop() + +def getrawcode(obj): + """ return code object for given function. """ + obj = getattr(obj, 'im_func', obj) + obj = getattr(obj, 'func_code', obj) + obj = getattr(obj, 'f_code', obj) + obj = getattr(obj, '__code__', obj) + return obj + Added: pypy/branch/py12/py/_code/oldmagic.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/oldmagic.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,62 @@ +""" deprecated module for turning on/off some features. """ + +import py + +from py.builtin import builtins as cpy_builtin + +def invoke(assertion=False, compile=False): + """ (deprecated) invoke magic, currently you can specify: + + assertion patches the builtin AssertionError to try to give + more meaningful AssertionErrors, which by means + of deploying a mini-interpreter constructs + a useful error message. + """ + py.log._apiwarn("1.1", + "py.magic.invoke() is deprecated, use py.code.patch_builtins()", + stacklevel=2, + ) + py.code.patch_builtins(assertion=assertion, compile=compile) + +def revoke(assertion=False, compile=False): + """ (deprecated) revoke previously invoked magic (see invoke()).""" + py.log._apiwarn("1.1", + "py.magic.revoke() is deprecated, use py.code.unpatch_builtins()", + stacklevel=2, + ) + py.code.unpatch_builtins(assertion=assertion, compile=compile) + +patched = {} + +def patch(namespace, name, value): + """ (deprecated) rebind the 'name' on the 'namespace' to the 'value', + possibly and remember the original value. Multiple + invocations to the same namespace/name pair will + remember a list of old values. + """ + py.log._apiwarn("1.1", + "py.magic.patch() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + orig = getattr(namespace, name) + patched.setdefault(nref, []).append(orig) + setattr(namespace, name, value) + return orig + +def revert(namespace, name): + """ (deprecated) revert to the orginal value the last patch modified. + Raise ValueError if no such original value exists. + """ + py.log._apiwarn("1.1", + "py.magic.revert() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + if nref not in patched or not patched[nref]: + raise ValueError("No original value stored for %s.%s" % nref) + current = getattr(namespace, name) + orig = patched[nref].pop() + setattr(namespace, name, orig) + return current + Added: pypy/branch/py12/py/_code/oldmagic2.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/oldmagic2.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,6 @@ + +import py + +py.log._apiwarn("1.1", "py.magic.AssertionError is deprecated, use py.code._AssertionError", stacklevel=2) + +from py.code import _AssertionError as AssertionError Added: pypy/branch/py12/py/_code/source.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_code/source.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,347 @@ +from __future__ import generators +import sys +import inspect, tokenize +import py +cpy_compile = compile + +try: + import _ast + from _ast import PyCF_ONLY_AST as _AST_FLAG +except ImportError: + _AST_FLAG = 0 + _ast = None + + +class Source(object): + """ a immutable object holding a source code fragment, + possibly deindenting it. + """ + def __init__(self, *parts, **kwargs): + self.lines = lines = [] + de = kwargs.get('deindent', True) + rstrip = kwargs.get('rstrip', True) + for part in parts: + if not part: + partlines = [] + if isinstance(part, Source): + partlines = part.lines + elif isinstance(part, py.builtin._basestring): + partlines = part.split('\n') + if rstrip: + while partlines: + if partlines[-1].strip(): + break + partlines.pop() + else: + partlines = getsource(part, deindent=de).lines + if de: + partlines = deindent(partlines) + lines.extend(partlines) + + def __eq__(self, other): + try: + return self.lines == other.lines + except AttributeError: + if isinstance(other, str): + return str(self) == other + return False + + def __getitem__(self, key): + if isinstance(key, int): + return self.lines[key] + else: + if key.step not in (None, 1): + raise IndexError("cannot slice a Source with a step") + return self.__getslice__(key.start, key.stop) + + def __len__(self): + return len(self.lines) + + def __getslice__(self, start, end): + newsource = Source() + newsource.lines = self.lines[start:end] + return newsource + + def strip(self): + """ return new source object with trailing + and leading blank lines removed. + """ + start, end = 0, len(self) + while start < end and not self.lines[start].strip(): + start += 1 + while end > start and not self.lines[end-1].strip(): + end -= 1 + source = Source() + source.lines[:] = self.lines[start:end] + return source + + def putaround(self, before='', after='', indent=' ' * 4): + """ return a copy of the source object with + 'before' and 'after' wrapped around it. + """ + before = Source(before) + after = Source(after) + newsource = Source() + lines = [ (indent + line) for line in self.lines] + newsource.lines = before.lines + lines + after.lines + return newsource + + def indent(self, indent=' ' * 4): + """ return a copy of the source object with + all lines indented by the given indent-string. + """ + newsource = Source() + newsource.lines = [(indent+line) for line in self.lines] + return newsource + + def getstatement(self, lineno): + """ return Source statement which contains the + given linenumber (counted from 0). + """ + start, end = self.getstatementrange(lineno) + return self[start:end] + + def getstatementrange(self, lineno): + """ return (start, end) tuple which spans the minimal + statement region which containing the given lineno. + """ + # XXX there must be a better than these heuristic ways ... + # XXX there may even be better heuristics :-) + if not (0 <= lineno < len(self)): + raise IndexError("lineno out of range") + + # 1. find the start of the statement + from codeop import compile_command + for start in range(lineno, -1, -1): + trylines = self.lines[start:lineno+1] + # quick hack to indent the source and get it as a string in one go + trylines.insert(0, 'def xxx():') + trysource = '\n '.join(trylines) + # ^ space here + try: + compile_command(trysource) + except (SyntaxError, OverflowError, ValueError): + pass + else: + break # got a valid or incomplete statement + + # 2. find the end of the statement + for end in range(lineno+1, len(self)+1): + trysource = self[start:end] + if trysource.isparseable(): + break + + return start, end + + def getblockend(self, lineno): + # XXX + lines = [x + '\n' for x in self.lines[lineno:]] + blocklines = inspect.getblock(lines) + #print blocklines + return lineno + len(blocklines) - 1 + + def deindent(self, offset=None): + """ return a new source object deindented by offset. + If offset is None then guess an indentation offset from + the first non-blank line. Subsequent lines which have a + lower indentation offset will be copied verbatim as + they are assumed to be part of multilines. + """ + # XXX maybe use the tokenizer to properly handle multiline + # strings etc.pp? + newsource = Source() + newsource.lines[:] = deindent(self.lines, offset) + return newsource + + def isparseable(self, deindent=True): + """ return True if source is parseable, heuristically + deindenting it by default. + """ + try: + import parser + except ImportError: + syntax_checker = lambda x: compile(x, 'asd', 'exec') + else: + syntax_checker = parser.suite + + if deindent: + source = str(self.deindent()) + else: + source = str(self) + try: + #compile(source+'\n', "x", "exec") + syntax_checker(source+'\n') + except SyntaxError: + return False + else: + return True + + def __str__(self): + return "\n".join(self.lines) + + def compile(self, filename=None, mode='exec', + flag=generators.compiler_flag, + dont_inherit=0, _genframe=None): + """ return compiled code object. if filename is None + invent an artificial filename which displays + the source/line position of the caller frame. + """ + if not filename or py.path.local(filename).check(file=0): + if _genframe is None: + _genframe = sys._getframe(1) # the caller + fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno + if not filename: + filename = '' % (fn, lineno) + else: + filename = '' % (filename, fn, lineno) + source = "\n".join(self.lines) + '\n' + try: + co = cpy_compile(source, filename, mode, flag) + except SyntaxError: + ex = sys.exc_info()[1] + # re-represent syntax errors from parsing python strings + msglines = self.lines[:ex.lineno] + if ex.offset: + msglines.append(" "*ex.offset + '^') + msglines.append("syntax error probably generated here: %s" % filename) + newex = SyntaxError('\n'.join(msglines)) + newex.offset = ex.offset + newex.lineno = ex.lineno + newex.text = ex.text + raise newex + else: + if flag & _AST_FLAG: + return co + co_filename = MyStr(filename) + co_filename.__source__ = self + return py.code.Code(co).new(rec=1, co_filename=co_filename) + #return newcode_withfilename(co, co_filename) + +# +# public API shortcut functions +# + +def compile_(source, filename=None, mode='exec', flags= + generators.compiler_flag, dont_inherit=0): + """ compile the given source to a raw code object, + which points back to the source code through + "co_filename.__source__". All code objects + contained in the code object will recursively + also have this special subclass-of-string + filename. + """ + if _ast is not None and isinstance(source, _ast.AST): + # XXX should Source support having AST? + return cpy_compile(source, filename, mode, flags, dont_inherit) + _genframe = sys._getframe(1) # the caller + s = Source(source) + co = s.compile(filename, mode, flags, _genframe=_genframe) + return co + + +def getfslineno(obj): + try: + code = py.code.Code(obj) + except TypeError: + # fallback to + fn = (py.std.inspect.getsourcefile(obj) or + py.std.inspect.getfile(obj)) + fspath = fn and py.path.local(fn) or None + if fspath: + try: + _, lineno = findsource(obj) + except IOError: + lineno = None + else: + lineno = None + else: + fspath = code.path + lineno = code.firstlineno + return fspath, lineno + +# +# helper functions +# +class MyStr(str): + """ custom string which allows to add attributes. """ + +def findsource(obj): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno + else: + lineno = obj.co_firstlineno - 1 + return fullsource, lineno + + +def getsource(obj, **kwargs): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) + else: + lineno = obj.co_firstlineno - 1 + end = fullsource.getblockend(lineno) + return Source(fullsource[lineno:end+1], deident=True) + + +def deindent(lines, offset=None): + if offset is None: + for line in lines: + line = line.expandtabs() + s = line.lstrip() + if s: + offset = len(line)-len(s) + break + else: + offset = 0 + if offset == 0: + return list(lines) + newlines = [] + def readline_generator(lines): + for line in lines: + yield line + '\n' + while True: + yield '' + + r = readline_generator(lines) + try: + readline = r.next + except AttributeError: + readline = r.__next__ + + try: + for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(readline): + if sline > len(lines): + break # End of input reached + if sline > len(newlines): + line = lines[sline - 1].expandtabs() + if line.lstrip() and line[:offset].isspace(): + line = line[offset:] # Deindent + newlines.append(line) + + for i in range(sline, eline): + # Don't deindent continuing lines of + # multiline tokens (i.e. multiline strings) + newlines.append(lines[i]) + except (IndentationError, tokenize.TokenError): + pass + # Add any lines we didn't see. E.g. if an exception was raised. + newlines.extend(lines[len(newlines):]) + return newlines Added: pypy/branch/py12/py/_compat/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_compat/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,2 @@ +""" compatibility modules (taken from 2.4.4) """ + Added: pypy/branch/py12/py/_compat/dep_doctest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_compat/dep_doctest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,5 @@ +import py + +py.log._apiwarn("1.1", "py.compat.doctest deprecated, use standard library version.", +stacklevel="apipkg") +doctest = py.std.doctest Added: pypy/branch/py12/py/_compat/dep_optparse.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_compat/dep_optparse.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,4 @@ +import py +py.log._apiwarn("1.1", "py.compat.optparse deprecated, use standard library version.", stacklevel="apipkg") + +optparse = py.std.optparse Added: pypy/branch/py12/py/_compat/dep_subprocess.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_compat/dep_subprocess.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,5 @@ + +import py +py.log._apiwarn("1.1", "py.compat.subprocess deprecated, use standard library version.", +stacklevel="apipkg") +subprocess = py.std.subprocess Added: pypy/branch/py12/py/_compat/dep_textwrap.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_compat/dep_textwrap.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,5 @@ +import py + +py.log._apiwarn("1.1", "py.compat.textwrap deprecated, use standard library version.", + stacklevel="apipkg") +textwrap = py.std.textwrap Added: pypy/branch/py12/py/_error.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_error.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,83 @@ +""" +create errno-specific classes for IO or os calls. + +""" +import sys, os, errno + +class Error(EnvironmentError): + def __repr__(self): + return "%s.%s %r: %s " %(self.__class__.__module__, + self.__class__.__name__, + self.__class__.__doc__, + " ".join(map(str, self.args)), + #repr(self.args) + ) + + def __str__(self): + s = "[%s]: %s" %(self.__class__.__doc__, + " ".join(map(str, self.args)), + ) + return s + +_winerrnomap = { + 2: errno.ENOENT, + 3: errno.ENOENT, + 17: errno.EEXIST, + 22: errno.ENOTDIR, + 267: errno.ENOTDIR, + 5: errno.EACCES, # anything better? +} + +class ErrorMaker(object): + """ lazily provides Exception classes for each possible POSIX errno + (as defined per the 'errno' module). All such instances + subclass EnvironmentError. + """ + Error = Error + _errno2class = {} + + def __getattr__(self, name): + eno = getattr(errno, name) + cls = self._geterrnoclass(eno) + setattr(self, name, cls) + return cls + + def _geterrnoclass(self, eno): + try: + return self._errno2class[eno] + except KeyError: + clsname = errno.errorcode.get(eno, "UnknownErrno%d" %(eno,)) + errorcls = type(Error)(clsname, (Error,), + {'__module__':'py.error', + '__doc__': os.strerror(eno)}) + self._errno2class[eno] = errorcls + return errorcls + + def checked_call(self, func, *args): + """ call a function and raise an errno-exception if applicable. """ + __tracebackhide__ = True + try: + return func(*args) + except self.Error: + raise + except EnvironmentError: + cls, value, tb = sys.exc_info() + if not hasattr(value, 'errno'): + raise + __tracebackhide__ = False + errno = value.errno + try: + if not isinstance(value, WindowsError): + raise NameError + except NameError: + # we are not on Windows, or we got a proper OSError + cls = self._geterrnoclass(errno) + else: + try: + cls = self._geterrnoclass(_winerrnomap[errno]) + except KeyError: + raise value + raise cls("%s%r" % (func.__name__, args)) + __tracebackhide__ = True + +error = ErrorMaker() Added: pypy/branch/py12/py/_io/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_io/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +""" input/output helping """ Added: pypy/branch/py12/py/_io/capture.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_io/capture.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,348 @@ +import os +import sys +import py +import tempfile + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +if sys.version_info < (3,0): + class TextIO(StringIO): + def write(self, data): + if not isinstance(data, unicode): + data = unicode(data, getattr(self, '_encoding', 'UTF-8')) + StringIO.write(self, data) +else: + TextIO = StringIO + +try: + from io import BytesIO +except ImportError: + class BytesIO(StringIO): + def write(self, data): + if isinstance(data, unicode): + raise TypeError("not a byte value: %r" %(data,)) + StringIO.write(self, data) + +class FDCapture: + """ Capture IO to/from a given os-level filedescriptor. """ + + def __init__(self, targetfd, tmpfile=None): + """ save targetfd descriptor, and open a new + temporary file there. If no tmpfile is + specified a tempfile.Tempfile() will be opened + in text mode. + """ + self.targetfd = targetfd + if tmpfile is None: + f = tempfile.TemporaryFile('wb+') + tmpfile = dupfile(f, encoding="UTF-8") + f.close() + self.tmpfile = tmpfile + self._savefd = os.dup(targetfd) + os.dup2(self.tmpfile.fileno(), targetfd) + self._patched = [] + + def setasfile(self, name, module=sys): + """ patch . to self.tmpfile + """ + key = (module, name) + self._patched.append((key, getattr(module, name))) + setattr(module, name, self.tmpfile) + + def unsetfiles(self): + """ unpatch all patched items + """ + while self._patched: + (module, name), value = self._patched.pop() + setattr(module, name, value) + + def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ + os.dup2(self._savefd, self.targetfd) + self.unsetfiles() + os.close(self._savefd) + self.tmpfile.seek(0) + return self.tmpfile + + def writeorg(self, data): + """ write a string to the original file descriptor + """ + tempfp = tempfile.TemporaryFile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(data) + finally: + tempfp.close() + + +def dupfile(f, mode=None, buffering=0, raising=False, encoding=None): + """ return a new open file object that's a duplicate of f + + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' + defines whether an exception is raised when an incompatible + file object is passed in (if raising is False, the file + object itself will be returned) + """ + try: + fd = f.fileno() + except AttributeError: + if raising: + raise + return f + newfd = os.dup(fd) + mode = mode and mode or f.mode + if sys.version_info >= (3,0): + if encoding is not None: + mode = mode.replace("b", "") + buffering = True + return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + else: + f = os.fdopen(newfd, mode, buffering) + if encoding is not None: + return EncodedFile(f, encoding) + return f + +class EncodedFile(object): + def __init__(self, _stream, encoding): + self._stream = _stream + self.encoding = encoding + + def write(self, obj): + if isinstance(obj, unicode): + obj = obj.encode(self.encoding) + elif isinstance(obj, str): + pass + else: + obj = str(obj) + self._stream.write(obj) + + def writelines(self, linelist): + data = ''.join(linelist) + self.write(data) + + def __getattr__(self, name): + return getattr(self._stream, name) + +class Capture(object): + def call(cls, func, *args, **kwargs): + """ return a (res, out, err) tuple where + out and err represent the output/error output + during function execution. + call the given function with args/kwargs + and capture output/error during its execution. + """ + so = cls() + try: + res = func(*args, **kwargs) + finally: + out, err = so.reset() + return res, out, err + call = classmethod(call) + + def reset(self): + """ reset sys.stdout/stderr and return captured output as strings. """ + if hasattr(self, '_suspended'): + outfile = self._kwargs['out'] + errfile = self._kwargs['err'] + del self._kwargs + else: + outfile, errfile = self.done() + out, err = "", "" + if outfile: + out = outfile.read() + outfile.close() + if errfile and errfile != outfile: + err = errfile.read() + errfile.close() + return out, err + + def suspend(self): + """ return current snapshot captures, memorize tempfiles. """ + assert not hasattr(self, '_suspended') + self._suspended = True + outerr = self.readouterr() + outfile, errfile = self.done() + self._kwargs['out'] = outfile + self._kwargs['err'] = errfile + return outerr + + def resume(self): + """ resume capturing with original temp files. """ + assert self._suspended + self._initialize(**self._kwargs) + del self._suspended + + +class StdCaptureFD(Capture): + """ This class allows to capture writes to FD1 and FD2 + and may connect a NULL file to FD0 (and prevent + reads from sys.stdin) + """ + def __init__(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + if in_: + self._oldin = (sys.stdin, os.dup(0)) + sys.stdin = DontReadFromInput() + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if out: + tmpfile = None + if hasattr(out, 'write'): + tmpfile = out + self.out = py.io.FDCapture(1, tmpfile=tmpfile) + if patchsys: + self.out.setasfile('stdout') + if err: + if mixed and out: + tmpfile = self.out.tmpfile + elif hasattr(err, 'write'): + tmpfile = err + else: + tmpfile = None + self.err = py.io.FDCapture(2, tmpfile=tmpfile) + if patchsys: + self.err.setasfile('stderr') + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + if hasattr(self, 'out'): + outfile = self.out.done() + else: + outfile = None + if hasattr(self, 'err'): + errfile = self.err.done() + else: + errfile = None + if hasattr(self, '_oldin'): + oldsys, oldfd = self._oldin + os.dup2(oldfd, 0) + os.close(oldfd) + sys.stdin = oldsys + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + l = [] + for name in ('out', 'err'): + res = "" + if hasattr(self, name): + f = getattr(self, name).tmpfile + f.seek(0) + res = f.read() + f.truncate(0) + f.seek(0) + l.append(res) + return l + +class StdCapture(Capture): + """ This class allows to capture writes to sys.stdout|stderr "in-memory" + and will raise errors on tries to read from sys.stdin. It only + modifies sys.stdout|stderr|stdin attributes and does not + touch underlying File Descriptors (use StdCaptureFD for that). + """ + def __init__(self, out=True, err=True, in_=True, mixed=False): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out, err, in_, mixed): + self._out = out + self._err = err + self._in = in_ + if out: + self._oldout = sys.stdout + if not hasattr(out, 'write'): + out = TextIO() + sys.stdout = self.out = out + if err: + self._olderr = sys.stderr + if out and mixed: + err = self.out + elif not hasattr(err, 'write'): + err = TextIO() + sys.stderr = self.err = err + if in_: + self._oldin = sys.stdin + sys.stdin = self.newin = DontReadFromInput() + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + o,e = sys.stdout, sys.stderr + if self._out: + try: + sys.stdout = self._oldout + except AttributeError: + raise IOError("stdout capturing already reset") + del self._oldout + outfile = self.out + outfile.seek(0) + else: + outfile = None + if self._err: + try: + sys.stderr = self._olderr + except AttributeError: + raise IOError("stderr capturing already reset") + del self._olderr + errfile = self.err + errfile.seek(0) + else: + errfile = None + if self._in: + sys.stdin = self._oldin + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + out = err = "" + if self._out: + out = sys.stdout.getvalue() + sys.stdout.truncate(0) + sys.stdout.seek(0) + if self._err: + err = sys.stderr.getvalue() + sys.stderr.truncate(0) + sys.stderr.seek(0) + return out, err + +class DontReadFromInput: + """Temporary stub class. Ideally when stdin is accessed, the + capturing should be turned off, with possibly all data captured + so far sent to the screen. This should be configurable, though, + because in automated test runs it is better to crash than + hang indefinitely. + """ + def read(self, *args): + raise IOError("reading from stdin while output is captured") + readline = read + readlines = read + __iter__ = read + + def fileno(self): + raise ValueError("redirected Stdin is pseudofile, has no fileno()") + def isatty(self): + return False + def close(self): + pass + +try: + devnullpath = os.devnull +except AttributeError: + if os.name == 'nt': + devnullpath = 'NUL' + else: + devnullpath = '/dev/null' + + Added: pypy/branch/py12/py/_io/saferepr.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_io/saferepr.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,56 @@ +import py +import sys, os.path + +builtin_repr = repr + +reprlib = py.builtin._tryimport('repr', 'reprlib') + +sysex = (KeyboardInterrupt, MemoryError, SystemExit) + +class SafeRepr(reprlib.Repr): + """ subclass of repr.Repr that limits the resulting size of repr() + and includes information on exceptions raised during the call. + """ + def repr(self, x): + return self._callhelper(reprlib.Repr.repr, self, x) + + def repr_instance(self, x, level): + return self._callhelper(builtin_repr, x) + + def _callhelper(self, call, x, *args): + try: + # Try the vanilla repr and make sure that the result is a string + s = call(x, *args) + except sysex: + raise + except: + cls, e, tb = sys.exc_info() + exc_name = getattr(cls, '__name__', 'unknown') + try: + exc_info = str(e) + except sysex: + raise + except: + exc_info = 'unknown' + return '<[%s("%s") raised in repr()] %s object at 0x%x>' % ( + exc_name, exc_info, x.__class__.__name__, id(x)) + else: + if len(s) > self.maxsize: + i = max(0, (self.maxsize-3)//2) + j = max(0, self.maxsize-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + +def saferepr(obj, maxsize=240): + """ return a size-limited safe repr-string for the given object. + Failing __repr__ functions of user instances will be represented + with a short exception info and 'saferepr' generally takes + care to never raise exceptions itself. This function is a wrapper + around the Repr/reprlib functionality of the standard 2.6 lib. + """ + # review exception handling + srepr = SafeRepr() + srepr.maxstring = maxsize + srepr.maxsize = maxsize + srepr.maxother = 160 + return srepr.repr(obj) Added: pypy/branch/py12/py/_io/terminalwriter.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_io/terminalwriter.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,273 @@ +""" + +Helper functions for writing to terminals and files. + +""" + + +import sys, os +import py + +def _getdimensions(): + import termios,fcntl,struct + call = fcntl.ioctl(0,termios.TIOCGWINSZ,"\000"*8) + height,width = struct.unpack( "hhhh", call ) [:2] + return height, width + +if sys.platform == 'win32': + # ctypes access to the Windows console + + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + FOREGROUND_BLUE = 0x0001 # text color contains blue. + FOREGROUND_GREEN = 0x0002 # text color contains green. + FOREGROUND_RED = 0x0004 # text color contains red. + FOREGROUND_WHITE = 0x0007 + FOREGROUND_INTENSITY = 0x0008 # text color is intensified. + BACKGROUND_BLUE = 0x0010 # background color contains blue. + BACKGROUND_GREEN = 0x0020 # background color contains green. + BACKGROUND_RED = 0x0040 # background color contains red. + BACKGROUND_WHITE = 0x0070 + BACKGROUND_INTENSITY = 0x0080 # background color is intensified. + + def GetStdHandle(kind): + import ctypes + return ctypes.windll.kernel32.GetStdHandle(kind) + + def SetConsoleTextAttribute(handle, attr): + import ctypes + ctypes.windll.kernel32.SetConsoleTextAttribute( + handle, attr) + + def _getdimensions(): + import ctypes + from ctypes import wintypes + + SHORT = ctypes.c_short + class COORD(ctypes.Structure): + _fields_ = [('X', SHORT), + ('Y', SHORT)] + class SMALL_RECT(ctypes.Structure): + _fields_ = [('Left', SHORT), + ('Top', SHORT), + ('Right', SHORT), + ('Bottom', SHORT)] + class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): + _fields_ = [('dwSize', COORD), + ('dwCursorPosition', COORD), + ('wAttributes', wintypes.WORD), + ('srWindow', SMALL_RECT), + ('dwMaximumWindowSize', COORD)] + STD_OUTPUT_HANDLE = -11 + handle = GetStdHandle(STD_OUTPUT_HANDLE) + info = CONSOLE_SCREEN_BUFFER_INFO() + ctypes.windll.kernel32.GetConsoleScreenBufferInfo( + handle, ctypes.byref(info)) + # Substract one from the width, otherwise the cursor wraps + # and the ending \n causes an empty line to display. + return info.dwSize.Y, info.dwSize.X - 1 + +def get_terminal_width(): + try: + height, width = _getdimensions() + except (SystemExit, KeyboardInterrupt): + raise + except: + # FALLBACK + width = int(os.environ.get('COLUMNS', 80)) + else: + # XXX the windows getdimensions may be bogus, let's sanify a bit + if width < 40: + width = 80 + return width + +terminal_width = get_terminal_width() + +# XXX unify with _escaped func below +def ansi_print(text, esc, file=None, newline=True, flush=False): + if file is None: + file = sys.stderr + text = text.rstrip() + if esc and not isinstance(esc, tuple): + esc = (esc,) + if esc and sys.platform != "win32" and file.isatty(): + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text + + '\x1b[0m') # ANSI color code "reset" + if newline: + text += '\n' + + if esc and sys.platform == "win32" and file.isatty(): + if 1 in esc: + bold = True + esc = tuple([x for x in esc if x != 1]) + else: + bold = False + esctable = {() : FOREGROUND_WHITE, # normal + (31,): FOREGROUND_RED, # red + (32,): FOREGROUND_GREEN, # green + (33,): FOREGROUND_GREEN|FOREGROUND_RED, # yellow + (34,): FOREGROUND_BLUE, # blue + (35,): FOREGROUND_BLUE|FOREGROUND_RED, # purple + (36,): FOREGROUND_BLUE|FOREGROUND_GREEN, # cyan + (37,): FOREGROUND_WHITE, # white + (39,): FOREGROUND_WHITE, # reset + } + attr = esctable.get(esc, FOREGROUND_WHITE) + if bold: + attr |= FOREGROUND_INTENSITY + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + if file is sys.stderr: + handle = GetStdHandle(STD_ERROR_HANDLE) + else: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + SetConsoleTextAttribute(handle, attr) + file.write(text) + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + else: + file.write(text) + + if flush: + file.flush() + +def should_do_markup(file): + return hasattr(file, 'isatty') and file.isatty() \ + and os.environ.get('TERM') != 'dumb' \ + and not (sys.platform.startswith('java') and os._name == 'nt') + +class TerminalWriter(object): + _esctable = dict(black=30, red=31, green=32, yellow=33, + blue=34, purple=35, cyan=36, white=37, + Black=40, Red=41, Green=42, Yellow=43, + Blue=44, Purple=45, Cyan=46, White=47, + bold=1, light=2, blink=5, invert=7) + + # XXX deprecate stringio argument + def __init__(self, file=None, stringio=False, encoding=None): + + if file is None: + if stringio: + self.stringio = file = py.io.TextIO() + else: + file = py.std.sys.stdout + if hasattr(file, 'encoding'): + encoding = file.encoding + elif hasattr(file, '__call__'): + file = WriteFile(file, encoding=encoding) + self.encoding = encoding + self._file = file + self.fullwidth = get_terminal_width() + self.hasmarkup = should_do_markup(file) + + def _escaped(self, text, esc): + if esc and self.hasmarkup: + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text +'\x1b[0m') + return text + + def markup(self, text, **kw): + esc = [] + for name in kw: + if name not in self._esctable: + raise ValueError("unknown markup: %r" %(name,)) + if kw[name]: + esc.append(self._esctable[name]) + return self._escaped(text, tuple(esc)) + + def sep(self, sepchar, title=None, fullwidth=None, **kw): + if fullwidth is None: + fullwidth = self.fullwidth + # the goal is to have the line be as long as possible + # under the condition that len(line) <= fullwidth + if title is not None: + # we want 2 + 2*len(fill) + len(title) <= fullwidth + # i.e. 2 + 2*len(sepchar)*N + len(title) <= fullwidth + # 2*len(sepchar)*N <= fullwidth - len(title) - 2 + # N <= (fullwidth - len(title) - 2) // (2*len(sepchar)) + N = (fullwidth - len(title) - 2) // (2*len(sepchar)) + fill = sepchar * N + line = "%s %s %s" % (fill, title, fill) + else: + # we want len(sepchar)*N <= fullwidth + # i.e. N <= fullwidth // len(sepchar) + line = sepchar * (fullwidth // len(sepchar)) + # in some situations there is room for an extra sepchar at the right, + # in particular if we consider that with a sepchar like "_ " the + # trailing space is not important at the end of the line + if len(line) + len(sepchar.rstrip()) <= fullwidth: + line += sepchar.rstrip() + + self.line(line, **kw) + + def write(self, s, **kw): + if s: + if not isinstance(self._file, WriteFile): + s = self._getbytestring(s) + if self.hasmarkup and kw: + s = self.markup(s, **kw) + self._file.write(s) + self._file.flush() + + def _getbytestring(self, s): + # XXX review this and the whole logic + if self.encoding and sys.version_info[0] < 3 and isinstance(s, unicode): + return s.encode(self.encoding) + elif not isinstance(s, str): + try: + return str(s) + except UnicodeEncodeError: + return "" % type(s).__name__ + return s + + def line(self, s='', **kw): + self.write(s, **kw) + self.write('\n') + +class Win32ConsoleWriter(TerminalWriter): + def write(self, s, **kw): + if s: + s = self._getbytestring(s) + if self.hasmarkup: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + + if self.hasmarkup and kw: + attr = 0 + if kw.pop('bold', False): + attr |= FOREGROUND_INTENSITY + + if kw.pop('red', False): + attr |= FOREGROUND_RED + elif kw.pop('blue', False): + attr |= FOREGROUND_BLUE + elif kw.pop('green', False): + attr |= FOREGROUND_GREEN + else: + attr |= FOREGROUND_WHITE + + SetConsoleTextAttribute(handle, attr) + self._file.write(s) + self._file.flush() + if self.hasmarkup: + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + + def line(self, s="", **kw): + self.write(s+"\n", **kw) + +if sys.platform == 'win32': + TerminalWriter = Win32ConsoleWriter + +class WriteFile(object): + def __init__(self, writemethod, encoding=None): + self.encoding = encoding + self._writemethod = writemethod + + def write(self, data): + if self.encoding: + data = data.encode(self.encoding) + self._writemethod(data) + + def flush(self): + return + + Added: pypy/branch/py12/py/_log/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_log/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,2 @@ +""" logging API ('producers' and 'consumers' connected via keywords) """ + Added: pypy/branch/py12/py/_log/log.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_log/log.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,186 @@ +""" +basic logging functionality based on a producer/consumer scheme. + +XXX implement this API: (maybe put it into slogger.py?) + + log = Logger( + info=py.log.STDOUT, + debug=py.log.STDOUT, + command=None) + log.info("hello", "world") + log.command("hello", "world") + + log = Logger(info=Logger(something=...), + debug=py.log.STDOUT, + command=None) +""" +import py, sys + +class Message(object): + def __init__(self, keywords, args): + self.keywords = keywords + self.args = args + + def content(self): + return " ".join(map(str, self.args)) + + def prefix(self): + return "[%s] " % (":".join(self.keywords)) + + def __str__(self): + return self.prefix() + self.content() + + +class Producer(object): + """ (deprecated) Log producer API which sends messages to be logged + to a 'consumer' object, which then prints them to stdout, + stderr, files, etc. Used extensively by PyPy-1.1. + """ + + Message = Message # to allow later customization + keywords2consumer = {} + + def __init__(self, keywords, keywordmapper=None, **kw): + if hasattr(keywords, 'split'): + keywords = tuple(keywords.split()) + self._keywords = keywords + if keywordmapper is None: + keywordmapper = default_keywordmapper + self._keywordmapper = keywordmapper + + def __repr__(self): + return "" % ":".join(self._keywords) + + def __getattr__(self, name): + if '_' in name: + raise AttributeError(name) + producer = self.__class__(self._keywords + (name,)) + setattr(self, name, producer) + return producer + + def __call__(self, *args): + """ write a message to the appropriate consumer(s) """ + func = self._keywordmapper.getconsumer(self._keywords) + if func is not None: + func(self.Message(self._keywords, args)) + +class KeywordMapper: + def __init__(self): + self.keywords2consumer = {} + + def getstate(self): + return self.keywords2consumer.copy() + def setstate(self, state): + self.keywords2consumer.clear() + self.keywords2consumer.update(state) + + def getconsumer(self, keywords): + """ return a consumer matching the given keywords. + + tries to find the most suitable consumer by walking, starting from + the back, the list of keywords, the first consumer matching a + keyword is returned (falling back to py.log.default) + """ + for i in range(len(keywords), 0, -1): + try: + return self.keywords2consumer[keywords[:i]] + except KeyError: + continue + return self.keywords2consumer.get('default', default_consumer) + + def setconsumer(self, keywords, consumer): + """ set a consumer for a set of keywords. """ + # normalize to tuples + if isinstance(keywords, str): + keywords = tuple(filter(None, keywords.split())) + elif hasattr(keywords, '_keywords'): + keywords = keywords._keywords + elif not isinstance(keywords, tuple): + raise TypeError("key %r is not a string or tuple" % (keywords,)) + if consumer is not None and not py.builtin.callable(consumer): + if not hasattr(consumer, 'write'): + raise TypeError( + "%r should be None, callable or file-like" % (consumer,)) + consumer = File(consumer) + self.keywords2consumer[keywords] = consumer + +def default_consumer(msg): + """ the default consumer, prints the message to stdout (using 'print') """ + sys.stderr.write(str(msg)+"\n") + +default_keywordmapper = KeywordMapper() + +def setconsumer(keywords, consumer): + default_keywordmapper.setconsumer(keywords, consumer) + +def setstate(state): + default_keywordmapper.setstate(state) +def getstate(): + return default_keywordmapper.getstate() + +# +# Consumers +# + +class File(object): + """ log consumer wrapping a file(-like) object """ + def __init__(self, f): + assert hasattr(f, 'write') + #assert isinstance(f, file) or not hasattr(f, 'open') + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + self._file.write(str(msg) + "\n") + if hasattr(self._file, 'flush'): + self._file.flush() + +class Path(object): + """ log consumer that opens and writes to a Path """ + def __init__(self, filename, append=False, + delayed_create=False, buffering=False): + self._append = append + self._filename = str(filename) + self._buffering = buffering + if not delayed_create: + self._openfile() + + def _openfile(self): + mode = self._append and 'a' or 'w' + f = open(self._filename, mode) + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + if not hasattr(self, "_file"): + self._openfile() + self._file.write(str(msg) + "\n") + if not self._buffering: + self._file.flush() + +def STDOUT(msg): + """ consumer that writes to sys.stdout """ + sys.stdout.write(str(msg)+"\n") + +def STDERR(msg): + """ consumer that writes to sys.stderr """ + sys.stderr.write(str(msg)+"\n") + +class Syslog: + """ consumer that writes to the syslog daemon """ + + def __init__(self, priority = None): + if priority is None: + priority = self.LOG_INFO + self.priority = priority + + def __call__(self, msg): + """ write a message to the log """ + py.std.syslog.syslog(self.priority, str(msg)) + +for _prio in "EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG".split(): + _prio = "LOG_" + _prio + try: + setattr(Syslog, _prio, getattr(py.std.syslog, _prio)) + except AttributeError: + pass Added: pypy/branch/py12/py/_log/warning.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_log/warning.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,76 @@ +import py, sys + +class DeprecationWarning(DeprecationWarning): + def __init__(self, msg, path, lineno): + self.msg = msg + self.path = path + self.lineno = lineno + def __repr__(self): + return "%s:%d: %s" %(self.path, self.lineno+1, self.msg) + def __str__(self): + return self.msg + +def _apiwarn(startversion, msg, stacklevel=2, function=None): + # below is mostly COPIED from python2.4/warnings.py's def warn() + # Get context information + if isinstance(stacklevel, str): + frame = sys._getframe(1) + level = 1 + found = frame.f_code.co_filename.find(stacklevel) != -1 + while frame: + co = frame.f_code + if co.co_filename.find(stacklevel) == -1: + if found: + stacklevel = level + break + else: + found = True + level += 1 + frame = frame.f_back + else: + stacklevel = 1 + msg = "%s (since version %s)" %(msg, startversion) + warn(msg, stacklevel=stacklevel+1, function=function) + +def warn(msg, stacklevel=1, function=None): + if function is not None: + filename = py.std.inspect.getfile(function) + lineno = py.code.getrawcode(function).co_firstlineno + else: + try: + caller = sys._getframe(stacklevel) + except ValueError: + globals = sys.__dict__ + lineno = 1 + else: + globals = caller.f_globals + lineno = caller.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + filename = globals.get('__file__') + if filename: + fnl = filename.lower() + if fnl.endswith(".pyc") or fnl.endswith(".pyo"): + filename = filename[:-1] + elif fnl.endswith("$py.class"): + filename = filename.replace('$py.class', '.py') + else: + if module == "__main__": + try: + filename = sys.argv[0] + except AttributeError: + # embedded interpreters don't have sys.argv, see bug #839151 + filename = '__main__' + if not filename: + filename = module + path = py.path.local(filename) + warning = DeprecationWarning(msg, path, lineno) + py.std.warnings.warn_explicit(warning, category=Warning, + filename=str(warning.path), + lineno=warning.lineno, + registry=py.std.warnings.__dict__.setdefault( + "__warningsregistry__", {}) + ) + Added: pypy/branch/py12/py/_path/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +""" unified file system api """ Added: pypy/branch/py12/py/_path/cacheutil.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/cacheutil.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,114 @@ +""" +This module contains multithread-safe cache implementations. + +All Caches have + + getorbuild(key, builder) + delentry(key) + +methods and allow configuration when instantiating the cache class. +""" +from time import time as gettime + +class BasicCache(object): + def __init__(self, maxentries=128): + self.maxentries = maxentries + self.prunenum = int(maxentries - maxentries/8) + self._dict = {} + + def clear(self): + self._dict.clear() + + def _getentry(self, key): + return self._dict[key] + + def _putentry(self, key, entry): + self._prunelowestweight() + self._dict[key] = entry + + def delentry(self, key, raising=False): + try: + del self._dict[key] + except KeyError: + if raising: + raise + + def getorbuild(self, key, builder): + try: + entry = self._getentry(key) + except KeyError: + entry = self._build(key, builder) + self._putentry(key, entry) + return entry.value + + def _prunelowestweight(self): + """ prune out entries with lowest weight. """ + numentries = len(self._dict) + if numentries >= self.maxentries: + # evict according to entry's weight + items = [(entry.weight, key) + for key, entry in self._dict.items()] + items.sort() + index = numentries - self.prunenum + if index > 0: + for weight, key in items[:index]: + # in MT situations the element might be gone + self.delentry(key, raising=False) + +class BuildcostAccessCache(BasicCache): + """ A BuildTime/Access-counting cache implementation. + the weight of a value is computed as the product of + + num-accesses-of-a-value * time-to-build-the-value + + The values with the least such weights are evicted + if the cache maxentries threshold is superceded. + For implementation flexibility more than one object + might be evicted at a time. + """ + # time function to use for measuring build-times + + def _build(self, key, builder): + start = gettime() + val = builder() + end = gettime() + return WeightedCountingEntry(val, end-start) + + +class WeightedCountingEntry(object): + def __init__(self, value, oneweight): + self._value = value + self.weight = self._oneweight = oneweight + + def value(self): + self.weight += self._oneweight + return self._value + value = property(value) + +class AgingCache(BasicCache): + """ This cache prunes out cache entries that are too old. + """ + def __init__(self, maxentries=128, maxseconds=10.0): + super(AgingCache, self).__init__(maxentries) + self.maxseconds = maxseconds + + def _getentry(self, key): + entry = self._dict[key] + if entry.isexpired(): + self.delentry(key) + raise KeyError(key) + return entry + + def _build(self, key, builder): + val = builder() + entry = AgingEntry(val, gettime() + self.maxseconds) + return entry + +class AgingEntry(object): + def __init__(self, value, expirationtime): + self.value = value + self.weight = expirationtime + + def isexpired(self): + t = gettime() + return t >= self.weight Added: pypy/branch/py12/py/_path/common.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/common.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,333 @@ +""" +""" +import os, sys +import py + +class Checkers: + _depend_on_existence = 'exists', 'link', 'dir', 'file' + + def __init__(self, path): + self.path = path + + def dir(self): + raise NotImplementedError + + def file(self): + raise NotImplementedError + + def dotfile(self): + return self.path.basename.startswith('.') + + def ext(self, arg): + if not arg.startswith('.'): + arg = '.' + arg + return self.path.ext == arg + + def exists(self): + raise NotImplementedError + + def basename(self, arg): + return self.path.basename == arg + + def basestarts(self, arg): + return self.path.basename.startswith(arg) + + def relto(self, arg): + return self.path.relto(arg) + + def fnmatch(self, arg): + return FNMatcher(arg)(self.path) + + def endswith(self, arg): + return str(self.path).endswith(arg) + + def _evaluate(self, kw): + for name, value in kw.items(): + invert = False + meth = None + try: + meth = getattr(self, name) + except AttributeError: + if name[:3] == 'not': + invert = True + try: + meth = getattr(self, name[3:]) + except AttributeError: + pass + if meth is None: + raise TypeError( + "no %r checker available for %r" % (name, self.path)) + try: + if py.code.getrawcode(meth).co_argcount > 1: + if (not meth(value)) ^ invert: + return False + else: + if bool(value) ^ bool(meth()) ^ invert: + return False + except (py.error.ENOENT, py.error.ENOTDIR): + for name in self._depend_on_existence: + if name in kw: + if kw.get(name): + return False + name = 'not' + name + if name in kw: + if not kw.get(name): + return False + return True + +class NeverRaised(Exception): + pass + +class PathBase(object): + """ shared implementation for filesystem path objects.""" + Checkers = Checkers + + def __div__(self, other): + return self.join(str(other)) + __truediv__ = __div__ # py3k + + def basename(self): + """ basename part of path. """ + return self._getbyspec('basename')[0] + basename = property(basename, None, None, basename.__doc__) + + def purebasename(self): + """ pure base name of the path.""" + return self._getbyspec('purebasename')[0] + purebasename = property(purebasename, None, None, purebasename.__doc__) + + def ext(self): + """ extension of the path (including the '.').""" + return self._getbyspec('ext')[0] + ext = property(ext, None, None, ext.__doc__) + + def dirpath(self, *args, **kwargs): + """ return the directory Path of the current Path joined + with any given path arguments. + """ + return self.new(basename='').join(*args, **kwargs) + + def read(self, mode='r'): + """ read and return a bytestring from reading the path. """ + if sys.version_info < (2,3): + for x in 'u', 'U': + if x in mode: + mode = mode.replace(x, '') + f = self.open(mode) + try: + return f.read() + finally: + f.close() + + def readlines(self, cr=1): + """ read and return a list of lines from the path. if cr is False, the +newline will be removed from the end of each line. """ + if not cr: + content = self.read('rU') + return content.split('\n') + else: + f = self.open('rU') + try: + return f.readlines() + finally: + f.close() + + def load(self): + """ (deprecated) return object unpickled from self.read() """ + f = self.open('rb') + try: + return py.error.checked_call(py.std.pickle.load, f) + finally: + f.close() + + def move(self, target): + """ move this path to target. """ + if target.relto(self): + raise py.error.EINVAL(target, + "cannot move path into a subdirectory of itself") + try: + self.rename(target) + except py.error.EXDEV: # invalid cross-device link + self.copy(target) + self.remove() + + def __repr__(self): + """ return a string representation of this path. """ + return repr(str(self)) + + def check(self, **kw): + """ check a path for existence, or query its properties + + without arguments, this returns True if the path exists (on the + filesystem), False if not + + with (keyword only) arguments, the object compares the value + of the argument with the value of a property with the same name + (if it has one, else it raises a TypeError) + + when for example the keyword argument 'ext' is '.py', this will + return True if self.ext == '.py', False otherwise + """ + if not kw: + kw = {'exists' : 1} + return self.Checkers(self)._evaluate(kw) + + def relto(self, relpath): + """ return a string which is the relative part of the path + to the given 'relpath'. + """ + if not isinstance(relpath, (str, PathBase)): + raise TypeError("%r: not a string or path object" %(relpath,)) + strrelpath = str(relpath) + if strrelpath and strrelpath[-1] != self.sep: + strrelpath += self.sep + #assert strrelpath[-1] == self.sep + #assert strrelpath[-2] != self.sep + strself = str(self) + if sys.platform == "win32" or getattr(os, '_name', None) == 'nt': + if os.path.normcase(strself).startswith( + os.path.normcase(strrelpath)): + return strself[len(strrelpath):] + elif strself.startswith(strrelpath): + return strself[len(strrelpath):] + return "" + + def bestrelpath(self, dest): + """ return a string which is a relative path from self + to dest such that self.join(bestrelpath) == dest and + if not such path can be determined return dest. + """ + try: + base = self.common(dest) + if not base: # can be the case on windows + return str(dest) + self2base = self.relto(base) + reldest = dest.relto(base) + if self2base: + n = self2base.count(self.sep) + 1 + else: + n = 0 + l = ['..'] * n + if reldest: + l.append(reldest) + target = dest.sep.join(l) + return target + except AttributeError: + return str(dest) + + + def parts(self, reverse=False): + """ return a root-first list of all ancestor directories + plus the path itself. + """ + current = self + l = [self] + while 1: + last = current + current = current.dirpath() + if last == current: + break + l.insert(0, current) + if reverse: + l.reverse() + return l + + def common(self, other): + """ return the common part shared with the other path + or None if there is no common part. + """ + last = None + for x, y in zip(self.parts(), other.parts()): + if x != y: + return last + last = x + return last + + def __add__(self, other): + """ return new path object with 'other' added to the basename""" + return self.new(basename=self.basename+str(other)) + + def __cmp__(self, other): + """ return sort value (-1, 0, +1). """ + try: + return cmp(self.strpath, other.strpath) + except AttributeError: + return cmp(str(self), str(other)) # self.path, other.path) + + def __lt__(self, other): + try: + return self.strpath < other.strpath + except AttributeError: + return str(self) < str(other) + + def visit(self, fil=None, rec=None, ignore=NeverRaised): + """ yields all paths below the current one + + fil is a filter (glob pattern or callable), if not matching the + path will not be yielded, defaulting to None (everything is + returned) + + rec is a filter (glob pattern or callable) that controls whether + a node is descended, defaulting to None + + ignore is an Exception class that is ignoredwhen calling dirlist() + on any of the paths (by default, all exceptions are reported) + """ + if isinstance(fil, str): + fil = FNMatcher(fil) + if rec: + if isinstance(rec, str): + rec = fnmatch(fil) + elif not hasattr(rec, '__call__'): + rec = None + try: + entries = self.listdir() + except ignore: + return + dirs = [p for p in entries + if p.check(dir=1) and (rec is None or rec(p))] + for subdir in dirs: + for p in subdir.visit(fil=fil, rec=rec, ignore=ignore): + yield p + for p in entries: + if fil is None or fil(p): + yield p + + def _sortlist(self, res, sort): + if sort: + if hasattr(sort, '__call__'): + res.sort(sort) + else: + res.sort() + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return self.strpath == str(other) + +class FNMatcher: + def __init__(self, pattern): + self.pattern = pattern + def __call__(self, path): + """return true if the basename/fullname matches the glob-'pattern'. + + * matches everything + ? matches any single character + [seq] matches any character in seq + [!seq] matches any char not in seq + + if the pattern contains a path-separator then the full path + is used for pattern matching and a '*' is prepended to the + pattern. + + if the pattern doesn't contain a path-separator the pattern + is only matched against the basename. + """ + pattern = self.pattern + if pattern.find(path.sep) == -1: + name = path.basename + else: + name = str(path) # path.strpath # XXX svn? + pattern = '*' + path.sep + pattern + from fnmatch import fnmatch + return fnmatch(name, pattern) + Added: pypy/branch/py12/py/_path/gateway/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/gateway/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +# Added: pypy/branch/py12/py/_path/gateway/channeltest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/gateway/channeltest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,65 @@ +import threading + + +class PathServer: + + def __init__(self, channel): + self.channel = channel + self.C2P = {} + self.next_id = 0 + threading.Thread(target=self.serve).start() + + def p2c(self, path): + id = self.next_id + self.next_id += 1 + self.C2P[id] = path + return id + + def command_LIST(self, id, *args): + path = self.C2P[id] + answer = [(self.p2c(p), p.basename) for p in path.listdir(*args)] + self.channel.send(answer) + + def command_DEL(self, id): + del self.C2P[id] + + def command_GET(self, id, spec): + path = self.C2P[id] + self.channel.send(path._getbyspec(spec)) + + def command_READ(self, id): + path = self.C2P[id] + self.channel.send(path.read()) + + def command_JOIN(self, id, resultid, *args): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.join(*args) + + def command_DIRPATH(self, id, resultid): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.dirpath() + + def serve(self): + try: + while 1: + msg = self.channel.receive() + meth = getattr(self, 'command_' + msg[0]) + meth(*msg[1:]) + except EOFError: + pass + +if __name__ == '__main__': + import py + gw = execnet.PopenGateway() + channel = gw._channelfactory.new() + srv = PathServer(channel) + c = gw.remote_exec(""" + import remotepath + p = remotepath.RemotePath(channel.receive(), channel.receive()) + channel.send(len(p.listdir())) + """) + c.send(channel) + c.send(srv.p2c(py.path.local('/tmp'))) + print(c.receive()) Added: pypy/branch/py12/py/_path/gateway/channeltest2.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/gateway/channeltest2.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,21 @@ +import py +from remotepath import RemotePath + + +SRC = open('channeltest.py', 'r').read() + +SRC += ''' +import py +srv = PathServer(channel.receive()) +channel.send(srv.p2c(py.path.local("/tmp"))) +''' + + +#gw = execnet.SshGateway('codespeak.net') +gw = execnet.PopenGateway() +gw.remote_init_threads(5) +c = gw.remote_exec(SRC, stdout=py.std.sys.stdout, stderr=py.std.sys.stderr) +subchannel = gw._channelfactory.new() +c.send(subchannel) + +p = RemotePath(subchannel, c.receive()) Added: pypy/branch/py12/py/_path/gateway/remotepath.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/gateway/remotepath.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,47 @@ +import py, itertools +from py._path import common + +COUNTER = itertools.count() + +class RemotePath(common.PathBase): + sep = '/' + + def __init__(self, channel, id, basename=None): + self._channel = channel + self._id = id + self._basename = basename + self._specs = {} + + def __del__(self): + self._channel.send(('DEL', self._id)) + + def __repr__(self): + return 'RemotePath(%s)' % self.basename + + def listdir(self, *args): + self._channel.send(('LIST', self._id) + args) + return [RemotePath(self._channel, id, basename) + for (id, basename) in self._channel.receive()] + + def dirpath(self): + id = ~COUNTER.next() + self._channel.send(('DIRPATH', self._id, id)) + return RemotePath(self._channel, id) + + def join(self, *args): + id = ~COUNTER.next() + self._channel.send(('JOIN', self._id, id) + args) + return RemotePath(self._channel, id) + + def _getbyspec(self, spec): + parts = spec.split(',') + ask = [x for x in parts if x not in self._specs] + if ask: + self._channel.send(('GET', self._id, ",".join(ask))) + for part, value in zip(ask, self._channel.receive()): + self._specs[part] = value + return [self._specs[x] for x in parts] + + def read(self): + self._channel.send(('READ', self._id)) + return self._channel.receive() Added: pypy/branch/py12/py/_path/local.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/local.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,809 @@ +""" +local path implementation. +""" +import sys, os, stat, re, atexit +import py +from py._path import common + +iswin32 = sys.platform == "win32" or (getattr(os, '_name', False) == 'nt') + +class Stat(object): + def __getattr__(self, name): + return getattr(self._osstatresult, "st_" + name) + + def __init__(self, path, osstatresult): + self.path = path + self._osstatresult = osstatresult + + def owner(self): + if iswin32: + raise NotImplementedError("XXX win32") + import pwd + entry = py.error.checked_call(pwd.getpwuid, self.uid) + return entry[0] + owner = property(owner, None, None, "owner of path") + + def group(self): + """ return group name of file. """ + if iswin32: + raise NotImplementedError("XXX win32") + import grp + entry = py.error.checked_call(grp.getgrgid, self.gid) + return entry[0] + group = property(group) + +class PosixPath(common.PathBase): + def chown(self, user, group, rec=0): + """ change ownership to the given user and group. + user and group may be specified by a number or + by a name. if rec is True change ownership + recursively. + """ + uid = getuserid(user) + gid = getgroupid(group) + if rec: + for x in self.visit(rec=lambda x: x.check(link=0)): + if x.check(link=0): + py.error.checked_call(os.chown, str(x), uid, gid) + py.error.checked_call(os.chown, str(self), uid, gid) + + def readlink(self): + """ return value of a symbolic link. """ + return py.error.checked_call(os.readlink, self.strpath) + + def mklinkto(self, oldname): + """ posix style hard link to another name. """ + py.error.checked_call(os.link, str(oldname), str(self)) + + def mksymlinkto(self, value, absolute=1): + """ create a symbolic link with the given value (pointing to another name). """ + if absolute: + py.error.checked_call(os.symlink, str(value), self.strpath) + else: + base = self.common(value) + # with posix local paths '/' is always a common base + relsource = self.__class__(value).relto(base) + reldest = self.relto(base) + n = reldest.count(self.sep) + target = self.sep.join(('..', )*n + (relsource, )) + py.error.checked_call(os.symlink, target, self.strpath) + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return py.error.checked_call(os.path.samefile, str(self), str(other)) + +def getuserid(user): + import pwd + if not isinstance(user, int): + user = pwd.getpwnam(user)[2] + return user + +def getgroupid(group): + import grp + if not isinstance(group, int): + group = grp.getgrnam(group)[2] + return group + +FSBase = not iswin32 and PosixPath or common.PathBase + +class LocalPath(FSBase): + """ object oriented interface to os.path and other local filesystem + related information. + """ + sep = os.sep + class Checkers(common.Checkers): + def _stat(self): + try: + return self._statcache + except AttributeError: + try: + self._statcache = self.path.stat() + except py.error.ELOOP: + self._statcache = self.path.lstat() + return self._statcache + + def dir(self): + return stat.S_ISDIR(self._stat().mode) + + def file(self): + return stat.S_ISREG(self._stat().mode) + + def exists(self): + return self._stat() + + def link(self): + st = self.path.lstat() + return stat.S_ISLNK(st.mode) + + def __new__(cls, path=None): + """ Initialize and return a local Path instance. + + Path can be relative to the current directory. + If it is None then the current working directory is taken. + Note that Path instances always carry an absolute path. + Note also that passing in a local path object will simply return + the exact same path object. Use new() to get a new copy. + """ + if isinstance(path, common.PathBase): + if path.__class__ == cls: + return path + path = path.strpath + # initialize the path + self = object.__new__(cls) + if not path: + self.strpath = os.getcwd() + elif isinstance(path, py.builtin._basestring): + self.strpath = os.path.abspath(os.path.normpath(str(path))) + else: + raise ValueError("can only pass None, Path instances " + "or non-empty strings to LocalPath") + assert isinstance(self.strpath, str) + return self + + def __hash__(self): + return hash(self.strpath) + + def __eq__(self, other): + s1 = str(self) + s2 = str(other) + if iswin32: + s1 = s1.lower() + s2 = s2.lower() + return s1 == s2 + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return str(self) < str(other) + + def remove(self, rec=1): + """ remove a file or directory (or a directory tree if rec=1). """ + if self.check(dir=1, link=0): + if rec: + # force remove of readonly files on windows + if iswin32: + self.chmod(448, rec=1) # octcal 0700 + py.error.checked_call(py.std.shutil.rmtree, self.strpath) + else: + py.error.checked_call(os.rmdir, self.strpath) + else: + if iswin32: + self.chmod(448) # octcal 0700 + py.error.checked_call(os.remove, self.strpath) + + def computehash(self, hashtype="md5", chunksize=524288): + """ return hexdigest of hashvalue for this file. """ + try: + try: + import hashlib as mod + except ImportError: + if hashtype == "sha1": + hashtype = "sha" + mod = __import__(hashtype) + hash = getattr(mod, hashtype)() + except (AttributeError, ImportError): + raise ValueError("Don't know how to compute %r hash" %(hashtype,)) + f = self.open('rb') + try: + while 1: + buf = f.read(chunksize) + if not buf: + return hash.hexdigest() + hash.update(buf) + finally: + f.close() + + def new(self, **kw): + """ create a modified version of this path. + the following keyword arguments modify various path parts: + + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + drive, dirname, basename, purebasename,ext = self._getbyspec( + "drive,dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + try: + ext = kw['ext'] + except KeyError: + pass + else: + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('drive', drive) + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + obj.strpath = os.path.normpath( + "%(drive)s%(dirname)s%(sep)s%(basename)s" % kw) + return obj + + def _getbyspec(self, spec): + """ return a sequence of specified path parts. 'spec' is + a comma separated string containing path part names. + according to the following convention: + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + + args = filter(None, spec.split(',') ) + append = res.append + for name in args: + if name == 'drive': + append(parts[0]) + elif name == 'dirname': + append(self.sep.join(['']+parts[1:-1])) + else: + basename = parts[-1] + if name == 'basename': + append(basename) + else: + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + append(purebasename) + elif name == 'ext': + append(ext) + else: + raise ValueError("invalid part specification %r" % name) + return res + + def join(self, *args, **kwargs): + """ return a new path by appending all 'args' as path + components. if abs=1 is used restart from root if any + of the args is an absolute path. + """ + if not args: + return self + strpath = self.strpath + sep = self.sep + strargs = [str(x) for x in args] + if kwargs.get('abs', 0): + for i in range(len(strargs)-1, -1, -1): + if os.path.isabs(strargs[i]): + strpath = strargs[i] + strargs = strargs[i+1:] + break + for arg in strargs: + arg = arg.strip(sep) + if iswin32: + # allow unix style paths even on windows. + arg = arg.strip('/') + arg = arg.replace('/', sep) + if arg: + if not strpath.endswith(sep): + strpath += sep + strpath += arg + obj = self.new() + obj.strpath = os.path.normpath(strpath) + return obj + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return py.error.checked_call(open, self.strpath, mode) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + res = [] + for name in py.error.checked_call(os.listdir, self.strpath): + childurl = self.join(name) + if fil is None or fil(childurl): + res.append(childurl) + self._sortlist(res, sort) + return res + + def size(self): + """ return size of the underlying file object """ + return self.stat().size + + def mtime(self): + """ return last modification time of the path. """ + return self.stat().mtime + + def copy(self, target, archive=False): + """ copy path to target.""" + assert not archive, "XXX archive-mode not supported" + if self.check(file=1): + if target.check(dir=1): + target = target.join(self.basename) + assert self!=target + copychunked(self, target) + else: + def rec(p): + return p.check(link=0) + for x in self.visit(rec=rec): + relpath = x.relto(self) + newx = target.join(relpath) + newx.dirpath().ensure(dir=1) + if x.check(link=1): + newx.mksymlinkto(x.readlink()) + elif x.check(file=1): + copychunked(x, newx) + elif x.check(dir=1): + newx.ensure(dir=1) + + def rename(self, target): + """ rename this path to target. """ + return py.error.checked_call(os.rename, str(self), str(target)) + + def dump(self, obj, bin=1): + """ pickle object into path location""" + f = self.open('wb') + try: + py.error.checked_call(py.std.pickle.dump, obj, f, bin) + finally: + f.close() + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + p = self.join(*args) + py.error.checked_call(os.mkdir, str(p)) + return p + + def write(self, data, mode='w'): + """ write data into path. """ + if 'b' in mode: + if not py.builtin._isbytes(data): + raise ValueError("can only process bytes") + else: + if not py.builtin._istext(data): + if not py.builtin._isbytes(data): + data = str(data) + else: + data = py.builtin._totext(data, sys.getdefaultencoding()) + f = self.open(mode) + try: + f.write(data) + finally: + f.close() + + def _ensuredirs(self): + parent = self.dirpath() + if parent == self: + return self + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + try: + self.mkdir() + except py.error.EEXIST: + # race condition: file/dir created by another thread/process. + # complain if it is not a dir + if self.check(dir=0): + raise + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if kwargs.get('dir', 0): + return p._ensuredirs() + else: + p.dirpath()._ensuredirs() + if not p.check(file=1): + p.open('w').close() + return p + + def stat(self): + """ Return an os.stat() tuple. """ + return Stat(self, py.error.checked_call(os.stat, self.strpath)) + + def lstat(self): + """ Return an os.lstat() tuple. """ + return Stat(self, py.error.checked_call(os.lstat, self.strpath)) + + def setmtime(self, mtime=None): + """ set modification time for the given path. if 'mtime' is None + (the default) then the file's mtime is set to current time. + + Note that the resolution for 'mtime' is platform dependent. + """ + if mtime is None: + return py.error.checked_call(os.utime, self.strpath, mtime) + try: + return py.error.checked_call(os.utime, self.strpath, (-1, mtime)) + except py.error.EINVAL: + return py.error.checked_call(os.utime, self.strpath, (self.atime(), mtime)) + + def chdir(self): + """ change directory to self and return old current directory """ + old = self.__class__() + py.error.checked_call(os.chdir, self.strpath) + return old + + def realpath(self): + """ return a new path which contains no symbolic links.""" + return self.__class__(os.path.realpath(self.strpath)) + + def atime(self): + """ return last access time of the path. """ + return self.stat().atime + + def __repr__(self): + return 'local(%r)' % self.strpath + + def __str__(self): + """ return string representation of the Path. """ + return self.strpath + + def pypkgpath(self, pkgname=None): + """ return the path's package path by looking for the given + pkgname. If pkgname is None then look for the last + directory upwards which still contains an __init__.py + and whose basename is python-importable. + Return None if a pkgpath can not be determined. + """ + pkgpath = None + for parent in self.parts(reverse=True): + if pkgname is None: + if parent.check(file=1): + continue + if not isimportable(parent.basename): + break + if parent.join('__init__.py').check(): + pkgpath = parent + continue + return pkgpath + else: + if parent.basename == pkgname: + return parent + return pkgpath + + def _prependsyspath(self, path): + s = str(path) + if s != sys.path[0]: + #print "prepending to sys.path", s + sys.path.insert(0, s) + + def chmod(self, mode, rec=0): + """ change permissions to the given mode. If mode is an + integer it directly encodes the os-specific modes. + if rec is True perform recursively. + """ + if not isinstance(mode, int): + raise TypeError("mode %r must be an integer" % (mode,)) + if rec: + for x in self.visit(rec=rec): + py.error.checked_call(os.chmod, str(x), mode) + py.error.checked_call(os.chmod, str(self), mode) + + def pyimport(self, modname=None, ensuresyspath=True): + """ return path as an imported python module. + if modname is None, look for the containing package + and construct an according module name. + The module will be put/looked up in sys.modules. + """ + if not self.check(): + raise py.error.ENOENT(self) + #print "trying to import", self + pkgpath = None + if modname is None: + pkgpath = self.pypkgpath() + if pkgpath is not None: + if ensuresyspath: + self._prependsyspath(pkgpath.dirpath()) + pkg = __import__(pkgpath.basename, None, None, []) + names = self.new(ext='').relto(pkgpath.dirpath()) + names = names.split(self.sep) + modname = ".".join(names) + else: + # no package scope, still make it possible + if ensuresyspath: + self._prependsyspath(self.dirpath()) + modname = self.purebasename + mod = __import__(modname, None, None, ['__doc__']) + modfile = mod.__file__ + if modfile[-4:] in ('.pyc', '.pyo'): + modfile = modfile[:-1] + elif modfile.endswith('$py.class'): + modfile = modfile[:-9] + '.py' + if not self.samefile(modfile): + raise EnvironmentError("mismatch:\n" + "imported module %r\n" + "does not stem from %r\n" + "maybe __init__.py files are missing?" % (mod, str(self))) + return mod + else: + try: + return sys.modules[modname] + except KeyError: + # we have a custom modname, do a pseudo-import + mod = py.std.types.ModuleType(modname) + mod.__file__ = str(self) + sys.modules[modname] = mod + try: + py.builtin.execfile(str(self), mod.__dict__) + except: + del sys.modules[modname] + raise + return mod + + def sysexec(self, *argv, **popen_opts): + """ return stdout text from executing a system child process, + where the 'self' path points to executable. + The process is directly invoked and not through a system shell. + """ + from subprocess import Popen, PIPE + argv = map(str, argv) + popen_opts['stdout'] = popen_opts['stderr'] = PIPE + proc = Popen([str(self)] + list(argv), **popen_opts) + stdout, stderr = proc.communicate() + ret = proc.wait() + if py.builtin._isbytes(stdout): + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + if ret != 0: + if py.builtin._isbytes(stderr): + stderr = py.builtin._totext(stderr, sys.getdefaultencoding()) + raise py.process.cmdexec.Error(ret, ret, str(self), + stdout, stderr,) + return stdout + + def sysfind(cls, name, checker=None): + """ return a path object found by looking at the systems + underlying PATH specification. If the checker is not None + it will be invoked to filter matching paths. If a binary + cannot be found, None is returned + Note: This is probably not working on plain win32 systems + but may work on cygwin. + """ + if os.path.isabs(name): + p = py.path.local(name) + if p.check(file=1): + return p + else: + if iswin32: + paths = py.std.os.environ['Path'].split(';') + if '' not in paths and '.' not in paths: + paths.append('.') + try: + systemroot = os.environ['SYSTEMROOT'] + except KeyError: + pass + else: + paths = [re.sub('%SystemRoot%', systemroot, path) + for path in paths] + tryadd = '', '.exe', '.com', '.bat' # XXX add more? + else: + paths = py.std.os.environ['PATH'].split(':') + tryadd = ('',) + + for x in paths: + for addext in tryadd: + p = py.path.local(x).join(name, abs=True) + addext + try: + if p.check(file=1): + if checker: + if not checker(p): + continue + return p + except py.error.EACCES: + pass + return None + sysfind = classmethod(sysfind) + + def _gethomedir(cls): + try: + x = os.environ['HOME'] + except KeyError: + x = os.environ['HOMEPATH'] + return cls(x) + _gethomedir = classmethod(_gethomedir) + + #""" + #special class constructors for local filesystem paths + #""" + def get_temproot(cls): + """ return the system's temporary directory + (where tempfiles are usually created in) + """ + return py.path.local(py.std.tempfile.gettempdir()) + get_temproot = classmethod(get_temproot) + + def mkdtemp(cls): + """ return a Path object pointing to a fresh new temporary directory + (which we created ourself). + """ + import tempfile + tries = 10 + for i in range(tries): + dname = tempfile.mktemp() + dpath = cls(tempfile.mktemp()) + try: + dpath.mkdir() + except (py.error.EEXIST, py.error.EPERM, py.error.EACCES): + continue + return dpath + raise py.error.ENOENT(dpath, "could not create tempdir, %d tries" % tries) + mkdtemp = classmethod(mkdtemp) + + def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800): # two days + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = cls.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError): # AttributeError on win32 + pass + + return udir + make_numbered_dir = classmethod(make_numbered_dir) + +def copychunked(src, dest): + chunksize = 524288 # half a meg of bytes + fsrc = src.open('rb') + try: + fdest = dest.open('wb') + try: + while 1: + buf = fsrc.read(chunksize) + if not buf: + break + fdest.write(buf) + finally: + fdest.close() + finally: + fsrc.close() + +def autopath(globs=None): + """ (deprecated) return the (local) path of the "current" file pointed to by globals or - if it is none - alternatively the callers frame globals. + + the path will always point to a .py file or to None. + the path will have the following payload: + pkgdir is the last parent directory path containing __init__.py + """ + py.log._apiwarn("1.1", "py.magic.autopath deprecated, " + "use py.path.local(__file__) and maybe pypkgpath/pyimport().") + if globs is None: + globs = sys._getframe(1).f_globals + try: + __file__ = globs['__file__'] + except KeyError: + if not sys.argv[0]: + raise ValueError("cannot compute autopath in interactive mode") + __file__ = os.path.abspath(sys.argv[0]) + + ret = py.path.local(__file__) + if ret.ext in ('.pyc', '.pyo'): + ret = ret.new(ext='.py') + current = pkgdir = ret.dirpath() + while 1: + if current.join('__init__.py').check(): + pkgdir = current + current = current.dirpath() + if pkgdir != current: + continue + elif str(current) not in sys.path: + sys.path.insert(0, str(current)) + break + ret.pkgdir = pkgdir + return ret + + +def isimportable(name): + if name: + if not (name[0].isalpha() or name[0] == '_'): + return False + name= name.replace("_", '') + return not name or name.isalnum() Added: pypy/branch/py12/py/_path/svnurl.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/svnurl.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,378 @@ +""" +module defining a subversion path object based on the external +command 'svn'. This modules aims to work with svn 1.3 and higher +but might also interact well with earlier versions. +""" + +import os, sys, time, re +import py +from py import path, process +from py._path import common +from py._path import svnwc as svncommon +from py._path.cacheutil import BuildcostAccessCache, AgingCache + +DEBUG=False + +class SvnCommandPath(svncommon.SvnPathBase): + """ path implementation that offers access to (possibly remote) subversion + repositories. """ + + _lsrevcache = BuildcostAccessCache(maxentries=128) + _lsnorevcache = AgingCache(maxentries=1000, maxseconds=60.0) + + def __new__(cls, path, rev=None, auth=None): + self = object.__new__(cls) + if isinstance(path, cls): + rev = path.rev + auth = path.auth + path = path.strpath + svncommon.checkbadchars(path) + path = path.rstrip('/') + self.strpath = path + self.rev = rev + self.auth = auth + return self + + def __repr__(self): + if self.rev == -1: + return 'svnurl(%r)' % self.strpath + else: + return 'svnurl(%r, %r)' % (self.strpath, self.rev) + + def _svnwithrev(self, cmd, *args): + """ execute an svn command, append our own url and revision """ + if self.rev is None: + return self._svnwrite(cmd, *args) + else: + args = ['-r', self.rev] + list(args) + return self._svnwrite(cmd, *args) + + def _svnwrite(self, cmd, *args): + """ execute an svn command, append our own url """ + l = ['svn %s' % cmd] + args = ['"%s"' % self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._encodedurl()) + # fixing the locale because we can't otherwise parse + string = " ".join(l) + if DEBUG: + print("execing %s" % string) + out = self._svncmdexecauth(string) + return out + + def _svncmdexecauth(self, cmd): + """ execute an svn command 'as is' """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._cmdexec(cmd) + + def _cmdexec(self, cmd): + try: + out = process.cmdexec(cmd) + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if (e.err.find('File Exists') != -1 or + e.err.find('File already exists') != -1): + raise py.error.EEXIST(self) + raise + return out + + def _svnpopenauth(self, cmd): + """ execute an svn command, return a pipe for reading stdin """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._popen(cmd) + + def _popen(self, cmd): + return os.popen(cmd) + + def _encodedurl(self): + return self._escape(self.strpath) + + def _norev_delentry(self, path): + auth = self.auth and self.auth.makecmdoptions() or None + self._lsnorevcache.delentry((str(path), auth)) + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + if mode not in ("r", "rU",): + raise ValueError("mode %r not supported" % (mode,)) + assert self.check(file=1) # svn cat returns an empty file otherwise + if self.rev is None: + return self._svnpopenauth('svn cat "%s"' % ( + self._escape(self.strpath), )) + else: + return self._svnpopenauth('svn cat -r %s "%s"' % ( + self.rev, self._escape(self.strpath))) + + def dirpath(self, *args, **kwargs): + """ return the directory path of the current path joined + with any given path arguments. + """ + l = self.strpath.split(self.sep) + if len(l) < 4: + raise py.error.EINVAL(self, "base is not valid") + elif len(l) == 4: + return self.join(*args, **kwargs) + else: + return self.new(basename='').join(*args, **kwargs) + + # modifying methods (cache must be invalidated) + def mkdir(self, *args, **kwargs): + """ create & return the directory joined with args. + pass a 'msg' keyword argument to set the commit message. + """ + commit_msg = kwargs.get('msg', "mkdir by py lib invocation") + createpath = self.join(*args) + createpath._svnwrite('mkdir', '-m', commit_msg) + self._norev_delentry(createpath.dirpath()) + return createpath + + def copy(self, target, msg='copied by py lib invocation'): + """ copy path to target with checkin message msg.""" + if getattr(target, 'rev', None) is not None: + raise py.error.EINVAL(target, "revisions are immutable") + self._svncmdexecauth('svn copy -m "%s" "%s" "%s"' %(msg, + self._escape(self), self._escape(target))) + self._norev_delentry(target.dirpath()) + + def rename(self, target, msg="renamed by py lib invocation"): + """ rename this path to target with checkin message msg. """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn move -m "%s" --force "%s" "%s"' %( + msg, self._escape(self), self._escape(target))) + self._norev_delentry(self.dirpath()) + self._norev_delentry(self) + + def remove(self, rec=1, msg='removed by py lib invocation'): + """ remove a file or directory (or a directory tree if rec=1) with +checkin message msg.""" + if self.rev is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn rm -m "%s" "%s"' %(msg, self._escape(self))) + self._norev_delentry(self.dirpath()) + + def export(self, topath): + """ export to a local path + + topath should not exist prior to calling this, returns a + py.path.local instance + """ + topath = py.path.local(topath) + args = ['"%s"' % (self._escape(self),), + '"%s"' % (self._escape(topath),)] + if self.rev is not None: + args = ['-r', str(self.rev)] + args + self._svncmdexecauth('svn export %s' % (' '.join(args),)) + return topath + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). If you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + target = self.join(*args) + dir = kwargs.get('dir', 0) + for x in target.parts(reverse=True): + if x.check(): + break + else: + raise py.error.ENOENT(target, "has not any valid base!") + if x == target: + if not x.check(dir=dir): + raise dir and py.error.ENOTDIR(x) or py.error.EISDIR(x) + return x + tocreate = target.relto(x) + basename = tocreate.split(self.sep, 1)[0] + tempdir = py.path.local.mkdtemp() + try: + tempdir.ensure(tocreate, dir=dir) + cmd = 'svn import -m "%s" "%s" "%s"' % ( + "ensure %s" % self._escape(tocreate), + self._escape(tempdir.join(basename)), + x.join(basename)._encodedurl()) + self._svncmdexecauth(cmd) + self._norev_delentry(x) + finally: + tempdir.remove() + return target + + # end of modifying methods + def _propget(self, name): + res = self._svnwithrev('propget', name) + return res[:-1] # strip trailing newline + + def _proplist(self): + res = self._svnwithrev('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return svncommon.PropListDict(self, lines) + + def info(self): + """ return an Info structure with svn-provided information. """ + parent = self.dirpath() + nameinfo_seq = parent._listdir_nameinfo() + bn = self.basename + for name, info in nameinfo_seq: + if name == bn: + return info + raise py.error.ENOENT(self) + + + def _listdir_nameinfo(self): + """ return sequence of name-info directory entries of self """ + def builder(): + try: + res = self._svnwithrev('ls', '-v') + except process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('non-existent in that revision') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('File not found') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('not part of a repository')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('Unable to open')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.lower().find('method not allowed')!=-1: + raise py.error.EACCES(self, e.err) + raise py.error.Error(e.err) + lines = res.split('\n') + nameinfo_seq = [] + for lsline in lines: + if lsline: + info = InfoSvnCommand(lsline) + if info._name != '.': # svn 1.5 produces '.' dirs, + nameinfo_seq.append((info._name, info)) + nameinfo_seq.sort() + return nameinfo_seq + auth = self.auth and self.auth.makecmdoptions() or None + if self.rev is not None: + return self._lsrevcache.getorbuild((self.strpath, self.rev, auth), + builder) + else: + return self._lsnorevcache.getorbuild((self.strpath, auth), + builder) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + nameinfo_seq = self._listdir_nameinfo() + if len(nameinfo_seq) == 1: + name, info = nameinfo_seq[0] + if name == self.basename and info.kind == 'file': + #if not self.check(dir=1): + raise py.error.ENOTDIR(self) + paths = [self.join(name) for (name, info) in nameinfo_seq] + if fil: + paths = [x for x in paths if fil(x)] + self._sortlist(paths, sort) + return paths + + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() #make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + xmlpipe = self._svnpopenauth('svn log --xml %s %s "%s"' % + (rev_opt, verbose_opt, self.strpath)) + from xml.dom import minidom + tree = minidom.parse(xmlpipe) + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(svncommon.LogEntry(logentry)) + return result + +#01234567890123456789012345678901234567890123467 +# 2256 hpk 165 Nov 24 17:55 __init__.py +# XXX spotted by Guido, SVN 1.3.0 has different aligning, breaks the code!!! +# 1312 johnny 1627 May 05 14:32 test_decorators.py +# +class InfoSvnCommand: + # the '0?' part in the middle is an indication of whether the resource is + # locked, see 'svn help ls' + lspattern = re.compile( + r'^ *(?P\d+) +(?P.+?) +(0? *(?P\d+))? ' + '*(?P\w+ +\d{2} +[\d:]+) +(?P.*)$') + def __init__(self, line): + # this is a typical line from 'svn ls http://...' + #_ 1127 jum 0 Jul 13 15:28 branch/ + match = self.lspattern.match(line) + data = match.groupdict() + self._name = data['file'] + if self._name[-1] == '/': + self._name = self._name[:-1] + self.kind = 'dir' + else: + self.kind = 'file' + #self.has_props = l.pop(0) == 'P' + self.created_rev = int(data['rev']) + self.last_author = data['author'] + self.size = data['size'] and int(data['size']) or 0 + self.mtime = parse_time_with_missing_year(data['date']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + +#____________________________________________________ +# +# helper functions +#____________________________________________________ +def parse_time_with_missing_year(timestr): + """ analyze the time part from a single line of "svn ls -v" + the svn output doesn't show the year makes the 'timestr' + ambigous. + """ + import calendar + t_now = time.gmtime() + + tparts = timestr.split() + month = time.strptime(tparts.pop(0), '%b')[1] + day = time.strptime(tparts.pop(0), '%d')[2] + last = tparts.pop(0) # year or hour:minute + try: + if ":" in last: + raise ValueError() + year = time.strptime(last, '%Y')[0] + hour = minute = 0 + except ValueError: + hour, minute = time.strptime(last, '%H:%M')[3:5] + year = t_now[0] + + t_result = (year, month, day, hour, minute, 0,0,0,0) + if t_result > t_now: + year -= 1 + t_result = (year, month, day, hour, minute, 0,0,0,0) + return calendar.timegm(t_result) + +class PathEntry: + def __init__(self, ppart): + self.strpath = ppart.firstChild.nodeValue.encode('UTF-8') + self.action = ppart.getAttribute('action').encode('UTF-8') + if self.action == 'A': + self.copyfrom_path = ppart.getAttribute('copyfrom-path').encode('UTF-8') + if self.copyfrom_path: + self.copyfrom_rev = int(ppart.getAttribute('copyfrom-rev')) + Added: pypy/branch/py12/py/_path/svnwc.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_path/svnwc.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,1231 @@ +""" +svn-Command based Implementation of a Subversion WorkingCopy Path. + + SvnWCCommandPath is the main class. + +""" + +import os, sys, time, re, calendar +import py +import subprocess +from py._path import common + +#----------------------------------------------------------- +# Caching latest repository revision and repo-paths +# (getting them is slow with the current implementations) +# +# XXX make mt-safe +#----------------------------------------------------------- + +class cache: + proplist = {} + info = {} + entries = {} + prop = {} + +class RepoEntry: + def __init__(self, url, rev, timestamp): + self.url = url + self.rev = rev + self.timestamp = timestamp + + def __str__(self): + return "repo: %s;%s %s" %(self.url, self.rev, self.timestamp) + +class RepoCache: + """ The Repocache manages discovered repository paths + and their revisions. If inside a timeout the cache + will even return the revision of the root. + """ + timeout = 20 # seconds after which we forget that we know the last revision + + def __init__(self): + self.repos = [] + + def clear(self): + self.repos = [] + + def put(self, url, rev, timestamp=None): + if rev is None: + return + if timestamp is None: + timestamp = time.time() + + for entry in self.repos: + if url == entry.url: + entry.timestamp = timestamp + entry.rev = rev + #print "set repo", entry + break + else: + entry = RepoEntry(url, rev, timestamp) + self.repos.append(entry) + #print "appended repo", entry + + def get(self, url): + now = time.time() + for entry in self.repos: + if url.startswith(entry.url): + if now < entry.timestamp + self.timeout: + #print "returning immediate Etrny", entry + return entry.url, entry.rev + return entry.url, -1 + return url, -1 + +repositories = RepoCache() + + +# svn support code + +ALLOWED_CHARS = "_ -/\\=$.~+%" #add characters as necessary when tested +if sys.platform == "win32": + ALLOWED_CHARS += ":" +ALLOWED_CHARS_HOST = ALLOWED_CHARS + '@:' + +def _getsvnversion(ver=[]): + try: + return ver[0] + except IndexError: + v = py.process.cmdexec("svn -q --version") + v.strip() + v = '.'.join(v.split('.')[:2]) + ver.append(v) + return v + +def _escape_helper(text): + text = str(text) + if py.std.sys.platform != 'win32': + text = str(text).replace('$', '\\$') + return text + +def _check_for_bad_chars(text, allowed_chars=ALLOWED_CHARS): + for c in str(text): + if c.isalnum(): + continue + if c in allowed_chars: + continue + return True + return False + +def checkbadchars(url): + # (hpk) not quite sure about the exact purpose, guido w.? + proto, uri = url.split("://", 1) + if proto != "file": + host, uripath = uri.split('/', 1) + # only check for bad chars in the non-protocol parts + if (_check_for_bad_chars(host, ALLOWED_CHARS_HOST) \ + or _check_for_bad_chars(uripath, ALLOWED_CHARS)): + raise ValueError("bad char in %r" % (url, )) + + +#_______________________________________________________________ + +class SvnPathBase(common.PathBase): + """ Base implementation for SvnPath implementations. """ + sep = '/' + + def _geturl(self): + return self.strpath + url = property(_geturl, None, None, "url of this svn-path.") + + def __str__(self): + """ return a string representation (including rev-number) """ + return self.strpath + + def __hash__(self): + return hash(self.strpath) + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + obj.rev = kw.get('rev', self.rev) + obj.auth = kw.get('auth', self.auth) + dirname, basename, purebasename, ext = self._getbyspec( + "dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + ext = kw.setdefault('ext', ext) + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + if kw['basename']: + obj.strpath = "%(dirname)s%(sep)s%(basename)s" % kw + else: + obj.strpath = "%(dirname)s" % kw + return obj + + def _getbyspec(self, spec): + """ get specified parts of the path. 'arg' is a string + with comma separated path parts. The parts are returned + in exactly the order of the specification. + + you may specify the following parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + for name in spec.split(','): + name = name.strip() + if name == 'dirname': + res.append(self.sep.join(parts[:-1])) + elif name == 'basename': + res.append(parts[-1]) + else: + basename = parts[-1] + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + res.append(purebasename) + elif name == 'ext': + res.append(ext) + else: + raise NameError("Don't know part %r" % name) + return res + + def __eq__(self, other): + """ return true if path and rev attributes each match """ + return (str(self) == str(other) and + (self.rev == other.rev or self.rev == other.rev)) + + def __ne__(self, other): + return not self == other + + def join(self, *args): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + + args = tuple([arg.strip(self.sep) for arg in args]) + parts = (self.strpath, ) + args + newpath = self.__class__(self.sep.join(parts), self.rev, self.auth) + return newpath + + def propget(self, name): + """ return the content of the given property. """ + value = self._propget(name) + return value + + def proplist(self): + """ list all property names. """ + content = self._proplist() + return content + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + # shared help methods + + def _escape(self, cmd): + return _escape_helper(cmd) + + + #def _childmaxrev(self): + # """ return maximum revision number of childs (or self.rev if no childs) """ + # rev = self.rev + # for name, info in self._listdir_nameinfo(): + # rev = max(rev, info.created_rev) + # return rev + + #def _getlatestrevision(self): + # """ return latest repo-revision for this path. """ + # url = self.strpath + # path = self.__class__(url, None) + # + # # we need a long walk to find the root-repo and revision + # while 1: + # try: + # rev = max(rev, path._childmaxrev()) + # previous = path + # path = path.dirpath() + # except (IOError, process.cmdexec.Error): + # break + # if rev is None: + # raise IOError, "could not determine newest repo revision for %s" % self + # return rev + + class Checkers(common.Checkers): + def dir(self): + try: + return self.path.info().kind == 'dir' + except py.error.Error: + return self._listdirworks() + + def _listdirworks(self): + try: + self.path.listdir() + except py.error.ENOENT: + return False + else: + return True + + def file(self): + try: + return self.path.info().kind == 'file' + except py.error.ENOENT: + return False + + def exists(self): + try: + return self.path.info() + except py.error.ENOENT: + return self._listdirworks() + +def parse_apr_time(timestr): + i = timestr.rfind('.') + if i == -1: + raise ValueError("could not parse %s" % timestr) + timestr = timestr[:i] + parsedtime = time.strptime(timestr, "%Y-%m-%dT%H:%M:%S") + return time.mktime(parsedtime) + +class PropListDict(dict): + """ a Dictionary which fetches values (InfoSvnCommand instances) lazily""" + def __init__(self, path, keynames): + dict.__init__(self, [(x, None) for x in keynames]) + self.path = path + + def __getitem__(self, key): + value = dict.__getitem__(self, key) + if value is None: + value = self.path.propget(key) + dict.__setitem__(self, key, value) + return value + +def fixlocale(): + if sys.platform != 'win32': + return 'LC_ALL=C ' + return '' + +# some nasty chunk of code to solve path and url conversion and quoting issues +ILLEGAL_CHARS = '* | \ / : < > ? \t \n \x0b \x0c \r'.split(' ') +if os.sep in ILLEGAL_CHARS: + ILLEGAL_CHARS.remove(os.sep) +ISWINDOWS = sys.platform == 'win32' +_reg_allow_disk = re.compile(r'^([a-z]\:\\)?[^:]+$', re.I) +def _check_path(path): + illegal = ILLEGAL_CHARS[:] + sp = path.strpath + if ISWINDOWS: + illegal.remove(':') + if not _reg_allow_disk.match(sp): + raise ValueError('path may not contain a colon (:)') + for char in sp: + if char not in string.printable or char in illegal: + raise ValueError('illegal character %r in path' % (char,)) + +def path_to_fspath(path, addat=True): + _check_path(path) + sp = path.strpath + if addat and path.rev != -1: + sp = '%s@%s' % (sp, path.rev) + elif addat: + sp = '%s at HEAD' % (sp,) + return sp + +def url_from_path(path): + fspath = path_to_fspath(path, False) + quote = py.std.urllib.quote + if ISWINDOWS: + match = _reg_allow_disk.match(fspath) + fspath = fspath.replace('\\', '/') + if match.group(1): + fspath = '/%s%s' % (match.group(1).replace('\\', '/'), + quote(fspath[len(match.group(1)):])) + else: + fspath = quote(fspath) + else: + fspath = quote(fspath) + if path.rev != -1: + fspath = '%s@%s' % (fspath, path.rev) + else: + fspath = '%s at HEAD' % (fspath,) + return 'file://%s' % (fspath,) + +class SvnAuth(object): + """ container for auth information for Subversion """ + def __init__(self, username, password, cache_auth=True, interactive=True): + self.username = username + self.password = password + self.cache_auth = cache_auth + self.interactive = interactive + + def makecmdoptions(self): + uname = self.username.replace('"', '\\"') + passwd = self.password.replace('"', '\\"') + ret = [] + if uname: + ret.append('--username="%s"' % (uname,)) + if passwd: + ret.append('--password="%s"' % (passwd,)) + if not self.cache_auth: + ret.append('--no-auth-cache') + if not self.interactive: + ret.append('--non-interactive') + return ' '.join(ret) + + def __str__(self): + return "" %(self.username,) + +rex_blame = re.compile(r'\s*(\d+)\s*(\S+) (.*)') + +class SvnWCCommandPath(common.PathBase): + """ path implementation offering access/modification to svn working copies. + It has methods similar to the functions in os.path and similar to the + commands of the svn client. + """ + sep = os.sep + + def __new__(cls, wcpath=None, auth=None): + self = object.__new__(cls) + if isinstance(wcpath, cls): + if wcpath.__class__ == cls: + return wcpath + wcpath = wcpath.localpath + if _check_for_bad_chars(str(wcpath), + ALLOWED_CHARS): + raise ValueError("bad char in wcpath %s" % (wcpath, )) + self.localpath = py.path.local(wcpath) + self.auth = auth + return self + + strpath = property(lambda x: str(x.localpath), None, None, "string path") + rev = property(lambda x: x.info(usecache=0).rev, None, None, "revision") + + def __eq__(self, other): + return self.localpath == getattr(other, 'localpath', None) + + def _geturl(self): + if getattr(self, '_url', None) is None: + info = self.info() + self._url = info.url #SvnPath(info.url, info.rev) + assert isinstance(self._url, py.builtin._basestring) + return self._url + + url = property(_geturl, None, None, "url of this WC item") + + def _escape(self, cmd): + return _escape_helper(cmd) + + def dump(self, obj): + """ pickle object into path location""" + return self.localpath.dump(obj) + + def svnurl(self): + """ return current SvnPath for this WC-item. """ + info = self.info() + return py.path.svnurl(info.url) + + def __repr__(self): + return "svnwc(%r)" % (self.strpath) # , self._url) + + def __str__(self): + return str(self.localpath) + + def _makeauthoptions(self): + if self.auth is None: + return '' + return self.auth.makecmdoptions() + + def _authsvn(self, cmd, args=None): + args = args and list(args) or [] + args.append(self._makeauthoptions()) + return self._svn(cmd, *args) + + def _svn(self, cmd, *args): + l = ['svn %s' % cmd] + args = [self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._escape(self.strpath)) + # try fixing the locale because we can't otherwise parse + string = fixlocale() + " ".join(l) + try: + try: + key = 'LC_MESSAGES' + hold = os.environ.get(key) + os.environ[key] = 'C' + out = py.process.cmdexec(string) + finally: + if hold: + os.environ[key] = hold + else: + del os.environ[key] + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + strerr = e.err.lower() + if strerr.find('file not found') != -1: + raise py.error.ENOENT(self) + if (strerr.find('file exists') != -1 or + strerr.find('file already exists') != -1 or + strerr.find("can't create directory") != -1): + raise py.error.EEXIST(self) + raise + return out + + def switch(self, url): + """ switch to given URL. """ + self._authsvn('switch', [url]) + + def checkout(self, url=None, rev=None): + """ checkout from url to local wcpath. """ + args = [] + if url is None: + url = self.url + if rev is None or rev == -1: + if (py.std.sys.platform != 'win32' and + _getsvnversion() == '1.3'): + url += "@HEAD" + else: + if _getsvnversion() == '1.3': + url += "@%d" % rev + else: + args.append('-r' + str(rev)) + args.append(url) + self._authsvn('co', args) + + def update(self, rev='HEAD', interactive=True): + """ update working copy item to given revision. (None -> HEAD). """ + opts = ['-r', rev] + if not interactive: + opts.append("--non-interactive") + self._authsvn('up', opts) + + def write(self, content, mode='w'): + """ write content into local filesystem wc. """ + self.localpath.write(content, mode) + + def dirpath(self, *args): + """ return the directory Path of the current Path. """ + return self.__class__(self.localpath.dirpath(*args), auth=self.auth) + + def _ensuredirs(self): + parent = self.dirpath() + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + self.mkdir() + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'directory=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if p.check(): + if p.check(versioned=False): + p.add() + return p + if kwargs.get('dir', 0): + return p._ensuredirs() + parent = p.dirpath() + parent._ensuredirs() + p.write("") + p.add() + return p + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + if args: + return self.join(*args).mkdir() + else: + self._svn('mkdir') + return self + + def add(self): + """ add ourself to svn """ + self._svn('add') + + def remove(self, rec=1, force=1): + """ remove a file or a directory tree. 'rec'ursive is + ignored and considered always true (because of + underlying svn semantics. + """ + assert rec, "svn cannot remove non-recursively" + if not self.check(versioned=True): + # not added to svn (anymore?), just remove + py.path.local(self).remove() + return + flags = [] + if force: + flags.append('--force') + self._svn('remove', *flags) + + def copy(self, target): + """ copy path to target.""" + py.process.cmdexec("svn copy %s %s" %(str(self), str(target))) + + def rename(self, target): + """ rename this path to target. """ + py.process.cmdexec("svn move --force %s %s" %(str(self), str(target))) + + def lock(self): + """ set a lock (exclusive) on the resource """ + out = self._authsvn('lock').strip() + if not out: + # warning or error, raise exception + raise Exception(out[4:]) + + def unlock(self): + """ unset a previously set lock """ + out = self._authsvn('unlock').strip() + if out.startswith('svn:'): + # warning or error, raise exception + raise Exception(out[4:]) + + def cleanup(self): + """ remove any locks from the resource """ + # XXX should be fixed properly!!! + try: + self.unlock() + except: + pass + + def status(self, updates=0, rec=0, externals=0): + """ return (collective) Status object for this file. """ + # http://svnbook.red-bean.com/book.html#svn-ch-3-sect-4.3.1 + # 2201 2192 jum test + # XXX + if externals: + raise ValueError("XXX cannot perform status() " + "on external items yet") + else: + #1.2 supports: externals = '--ignore-externals' + externals = '' + if rec: + rec= '' + else: + rec = '--non-recursive' + + # XXX does not work on all subversion versions + #if not externals: + # externals = '--ignore-externals' + + if updates: + updates = '-u' + else: + updates = '' + + try: + cmd = 'status -v --xml --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + except py.process.cmdexec.Error: + cmd = 'status -v --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + rootstatus = WCStatus(self).fromstring(out, self) + else: + rootstatus = XMLWCStatus(self).fromstring(out, self) + return rootstatus + + def diff(self, rev=None): + """ return a diff of the current path against revision rev (defaulting + to the last one). + """ + args = [] + if rev is not None: + args.append("-r %d" % rev) + out = self._authsvn('diff', args) + return out + + def blame(self): + """ return a list of tuples of three elements: + (revision, commiter, line) + """ + out = self._svn('blame') + result = [] + blamelines = out.splitlines() + reallines = py.path.svnurl(self.url).readlines() + for i, (blameline, line) in enumerate( + zip(blamelines, reallines)): + m = rex_blame.match(blameline) + if not m: + raise ValueError("output line %r of svn blame does not match " + "expected format" % (line, )) + rev, name, _ = m.groups() + result.append((int(rev), name, line)) + return result + + _rex_commit = re.compile(r'.*Committed revision (\d+)\.$', re.DOTALL) + def commit(self, msg='', rec=1): + """ commit with support for non-recursive commits """ + # XXX i guess escaping should be done better here?!? + cmd = 'commit -m "%s" --force-log' % (msg.replace('"', '\\"'),) + if not rec: + cmd += ' -N' + out = self._authsvn(cmd) + try: + del cache.info[self] + except KeyError: + pass + if out: + m = self._rex_commit.match(out) + return int(m.group(1)) + + def propset(self, name, value, *args): + """ set property name to value on this path. """ + d = py.path.local.mkdtemp() + try: + p = d.join('value') + p.write(value) + self._svn('propset', name, '--file', str(p), *args) + finally: + d.remove() + + def propget(self, name): + """ get property name on this path. """ + res = self._svn('propget', name) + return res[:-1] # strip trailing newline + + def propdel(self, name): + """ delete property name on this path. """ + res = self._svn('propdel', name) + return res[:-1] # strip trailing newline + + def proplist(self, rec=0): + """ return a mapping of property names to property values. +If rec is True, then return a dictionary mapping sub-paths to such mappings. +""" + if rec: + res = self._svn('proplist -R') + return make_recursive_propdict(self, res) + else: + res = self._svn('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return PropListDict(self, lines) + + def revert(self, rec=0): + """ revert the local changes of this path. if rec is True, do so +recursively. """ + if rec: + result = self._svn('revert -R') + else: + result = self._svn('revert') + return result + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + if kw: + localpath = self.localpath.new(**kw) + else: + localpath = self.localpath + return self.__class__(localpath, auth=self.auth) + + def join(self, *args, **kwargs): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + localpath = self.localpath.join(*args, **kwargs) + return self.__class__(localpath, auth=self.auth) + + def info(self, usecache=1): + """ return an Info structure with svn-provided information. """ + info = usecache and cache.info.get(self) + if not info: + try: + output = self._svn('info') + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('Path is not a working copy directory') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find("is not under version control") != -1: + raise py.error.ENOENT(self, e.err) + raise + # XXX SVN 1.3 has output on stderr instead of stdout (while it does + # return 0!), so a bit nasty, but we assume no output is output + # to stderr... + if (output.strip() == '' or + output.lower().find('not a versioned resource') != -1): + raise py.error.ENOENT(self, output) + info = InfoSvnWCCommand(output) + + # Can't reliably compare on Windows without access to win32api + if py.std.sys.platform != 'win32': + if info.path != self.localpath: + raise py.error.ENOENT(self, "not a versioned resource:" + + " %s != %s" % (info.path, self.localpath)) + cache.info[self] = info + return info + + def listdir(self, fil=None, sort=None): + """ return a sequence of Paths. + + listdir will return either a tuple or a list of paths + depending on implementation choices. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + # XXX unify argument naming with LocalPath.listdir + def notsvn(path): + return path.basename != '.svn' + + paths = [] + for localpath in self.localpath.listdir(notsvn): + p = self.__class__(localpath, auth=self.auth) + if notsvn(p) and (not fil or fil(p)): + paths.append(p) + self._sortlist(paths, sort) + return paths + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return open(self.strpath, mode) + + def _getbyspec(self, spec): + return self.localpath._getbyspec(spec) + + class Checkers(py.path.local.Checkers): + def __init__(self, path): + self.svnwcpath = path + self.path = path.localpath + def versioned(self): + try: + s = self.svnwcpath.info() + except (py.error.ENOENT, py.error.EEXIST): + return False + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('is not a working copy')!=-1: + return False + if e.err.lower().find('not a versioned resource') != -1: + return False + raise + else: + return True + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() # make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + locale_env = fixlocale() + # some blather on stderr + auth_opt = self._makeauthoptions() + #stdin, stdout, stderr = os.popen3(locale_env + + # 'svn log --xml %s %s %s "%s"' % ( + # rev_opt, verbose_opt, auth_opt, + # self.strpath)) + cmd = locale_env + 'svn log --xml %s %s %s "%s"' % ( + rev_opt, verbose_opt, auth_opt, self.strpath) + + popen = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + stdout, stderr = popen.communicate() + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + minidom,ExpatError = importxml() + try: + tree = minidom.parseString(stdout) + except ExpatError: + raise ValueError('no such revision') + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(LogEntry(logentry)) + return result + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + def __hash__(self): + return hash((self.strpath, self.__class__, self.auth)) + + +class WCStatus: + attrnames = ('modified','added', 'conflict', 'unchanged', 'external', + 'deleted', 'prop_modified', 'unknown', 'update_available', + 'incomplete', 'kindmismatch', 'ignored', 'locked', 'replaced' + ) + + def __init__(self, wcpath, rev=None, modrev=None, author=None): + self.wcpath = wcpath + self.rev = rev + self.modrev = modrev + self.author = author + + for name in self.attrnames: + setattr(self, name, []) + + def allpath(self, sort=True, **kw): + d = {} + for name in self.attrnames: + if name not in kw or kw[name]: + for path in getattr(self, name): + d[path] = 1 + l = d.keys() + if sort: + l.sort() + return l + + # XXX a bit scary to assume there's always 2 spaces between username and + # path, however with win32 allowing spaces in user names there doesn't + # seem to be a more solid approach :( + _rex_status = re.compile(r'\s+(\d+|-)\s+(\S+)\s+(.+?)\s{2,}(.*)') + + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ return a new WCStatus object from data 's' + """ + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + for line in data.split('\n'): + if not line.strip(): + continue + #print "processing %r" % line + flags, rest = line[:8], line[8:] + # first column + c0,c1,c2,c3,c4,c5,x6,c7 = flags + #if '*' in line: + # print "flags", repr(flags), "rest", repr(rest) + + if c0 in '?XI': + fn = line.split(None, 1)[1] + if c0 == '?': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.unknown.append(wcpath) + elif c0 == 'X': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(fn, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + elif c0 == 'I': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.ignored.append(wcpath) + + continue + + #elif c0 in '~!' or c4 == 'S': + # raise NotImplementedError("received flag %r" % c0) + + m = WCStatus._rex_status.match(rest) + if not m: + if c7 == '*': + fn = rest.strip() + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.update_available.append(wcpath) + continue + if line.lower().find('against revision:')!=-1: + update_rev = int(rest.split(':')[1].strip()) + continue + if line.lower().find('status on external') > -1: + # XXX not sure what to do here... perhaps we want to + # store some state instead of just continuing, as right + # now it makes the top-level external get added twice + # (once as external, once as 'normal' unchanged item) + # because of the way SVN presents external items + continue + # keep trying + raise ValueError("could not parse line %r" % line) + else: + rev, modrev, author, fn = m.groups() + wcpath = rootwcpath.join(fn, abs=1) + #assert wcpath.check() + if c0 == 'M': + assert wcpath.check(file=1), "didn't expect a directory with changed content here" + rootstatus.modified.append(wcpath) + elif c0 == 'A' or c3 == '+' : + rootstatus.added.append(wcpath) + elif c0 == 'D': + rootstatus.deleted.append(wcpath) + elif c0 == 'C': + rootstatus.conflict.append(wcpath) + elif c0 == '~': + rootstatus.kindmismatch.append(wcpath) + elif c0 == '!': + rootstatus.incomplete.append(wcpath) + elif c0 == 'R': + rootstatus.replaced.append(wcpath) + elif not c0.strip(): + rootstatus.unchanged.append(wcpath) + else: + raise NotImplementedError("received flag %r" % c0) + + if c1 == 'M': + rootstatus.prop_modified.append(wcpath) + # XXX do we cover all client versions here? + if c2 == 'L' or c5 == 'K': + rootstatus.locked.append(wcpath) + if c7 == '*': + rootstatus.update_available.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + if update_rev: + rootstatus.update_rev = update_rev + continue + return rootstatus + fromstring = staticmethod(fromstring) + +class XMLWCStatus(WCStatus): + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ parse 'data' (XML string as outputted by svn st) into a status obj + """ + # XXX for externals, the path is shown twice: once + # with external information, and once with full info as if + # the item was a normal non-external... the current way of + # dealing with this issue is by ignoring it - this does make + # externals appear as external items as well as 'normal', + # unchanged ones in the status object so this is far from ideal + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + minidom, ExpatError = importxml() + try: + doc = minidom.parseString(data) + except ExpatError: + e = sys.exc_info()[1] + raise ValueError(str(e)) + urevels = doc.getElementsByTagName('against') + if urevels: + rootstatus.update_rev = urevels[-1].getAttribute('revision') + for entryel in doc.getElementsByTagName('entry'): + path = entryel.getAttribute('path') + statusel = entryel.getElementsByTagName('wc-status')[0] + itemstatus = statusel.getAttribute('item') + + if itemstatus == 'unversioned': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.unknown.append(wcpath) + continue + elif itemstatus == 'external': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(path, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + continue + elif itemstatus == 'ignored': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.ignored.append(wcpath) + continue + elif itemstatus == 'incomplete': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.incomplete.append(wcpath) + continue + + rev = statusel.getAttribute('revision') + if itemstatus == 'added' or itemstatus == 'none': + rev = '0' + modrev = '?' + author = '?' + date = '' + else: + #print entryel.toxml() + commitel = entryel.getElementsByTagName('commit')[0] + if commitel: + modrev = commitel.getAttribute('revision') + author = '' + author_els = commitel.getElementsByTagName('author') + if author_els: + for c in author_els[0].childNodes: + author += c.nodeValue + date = '' + for c in commitel.getElementsByTagName('date')[0]\ + .childNodes: + date += c.nodeValue + + wcpath = rootwcpath.join(path, abs=1) + + assert itemstatus != 'modified' or wcpath.check(file=1), ( + 'did\'t expect a directory with changed content here') + + itemattrname = { + 'normal': 'unchanged', + 'unversioned': 'unknown', + 'conflicted': 'conflict', + 'none': 'added', + }.get(itemstatus, itemstatus) + + attr = getattr(rootstatus, itemattrname) + attr.append(wcpath) + + propsstatus = statusel.getAttribute('props') + if propsstatus not in ('none', 'normal'): + rootstatus.prop_modified.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + rootstatus.date = date + + # handle repos-status element (remote info) + rstatusels = entryel.getElementsByTagName('repos-status') + if rstatusels: + rstatusel = rstatusels[0] + ritemstatus = rstatusel.getAttribute('item') + if ritemstatus in ('added', 'modified'): + rootstatus.update_available.append(wcpath) + + lockels = entryel.getElementsByTagName('lock') + if len(lockels): + rootstatus.locked.append(wcpath) + + return rootstatus + fromstring = staticmethod(fromstring) + +class InfoSvnWCCommand: + def __init__(self, output): + # Path: test + # URL: http://codespeak.net/svn/std.path/trunk/dist/std.path/test + # Repository UUID: fd0d7bf2-dfb6-0310-8d31-b7ecfe96aada + # Revision: 2151 + # Node Kind: directory + # Schedule: normal + # Last Changed Author: hpk + # Last Changed Rev: 2100 + # Last Changed Date: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + # Properties Last Updated: 2003-11-03 14:47:48 +0100 (Mon, 03 Nov 2003) + + d = {} + for line in output.split('\n'): + if not line.strip(): + continue + key, value = line.split(':', 1) + key = key.lower().replace(' ', '') + value = value.strip() + d[key] = value + try: + self.url = d['url'] + except KeyError: + raise ValueError("Not a versioned resource") + #raise ValueError, "Not a versioned resource %r" % path + self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind'] + self.rev = int(d['revision']) + self.path = py.path.local(d['path']) + self.size = self.path.size() + if 'lastchangedrev' in d: + self.created_rev = int(d['lastchangedrev']) + if 'lastchangedauthor' in d: + self.last_author = d['lastchangedauthor'] + if 'lastchangeddate' in d: + self.mtime = parse_wcinfotime(d['lastchangeddate']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + +def parse_wcinfotime(timestr): + """ Returns seconds since epoch, UTC. """ + # example: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + m = re.match(r'(\d+-\d+-\d+ \d+:\d+:\d+) ([+-]\d+) .*', timestr) + if not m: + raise ValueError("timestring %r does not match" % timestr) + timestr, timezone = m.groups() + # do not handle timezone specially, return value should be UTC + parsedtime = time.strptime(timestr, "%Y-%m-%d %H:%M:%S") + return calendar.timegm(parsedtime) + +def make_recursive_propdict(wcroot, + output, + rex = re.compile("Properties on '(.*)':")): + """ Return a dictionary of path->PropListDict mappings. """ + lines = [x for x in output.split('\n') if x] + pdict = {} + while lines: + line = lines.pop(0) + m = rex.match(line) + if not m: + raise ValueError("could not parse propget-line: %r" % line) + path = m.groups()[0] + wcpath = wcroot.join(path, abs=1) + propnames = [] + while lines and lines[0].startswith(' '): + propname = lines.pop(0).strip() + propnames.append(propname) + assert propnames, "must have found properties!" + pdict[wcpath] = PropListDict(wcpath, propnames) + return pdict + + +def importxml(cache=[]): + if cache: + return cache + from xml.dom import minidom + from xml.parsers.expat import ExpatError + cache.extend([minidom, ExpatError]) + return cache + +class LogEntry: + def __init__(self, logentry): + self.rev = int(logentry.getAttribute('revision')) + for lpart in filter(None, logentry.childNodes): + if lpart.nodeType == lpart.ELEMENT_NODE: + if lpart.nodeName == 'author': + self.author = lpart.firstChild.nodeValue + elif lpart.nodeName == 'msg': + if lpart.firstChild: + self.msg = lpart.firstChild.nodeValue + else: + self.msg = '' + elif lpart.nodeName == 'date': + #2003-07-29T20:05:11.598637Z + timestr = lpart.firstChild.nodeValue + self.date = parse_apr_time(timestr) + elif lpart.nodeName == 'paths': + self.strpaths = [] + for ppart in filter(None, lpart.childNodes): + if ppart.nodeType == ppart.ELEMENT_NODE: + self.strpaths.append(PathEntry(ppart)) + def __repr__(self): + return '' % ( + self.rev, self.author, self.date) + + Added: pypy/branch/py12/py/_plugin/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +# Added: pypy/branch/py12/py/_plugin/hookspec.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/hookspec.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,172 @@ +""" +hook specifications for py.test plugins +""" + +# ------------------------------------------------------------------------- +# Command line and configuration +# ------------------------------------------------------------------------- + +def pytest_addoption(parser): + """ called before commandline parsing. """ + +def pytest_registerhooks(pluginmanager): + """ called after commandline parsing before pytest_configure. """ + +def pytest_namespace(): + """ return dict of name->object which will get stored at py.test. namespace""" + +def pytest_configure(config): + """ called after command line options have been parsed. + and all plugins and initial conftest files been loaded. + """ + +def pytest_unconfigure(config): + """ called before test process is exited. """ + +# ------------------------------------------------------------------------- +# collection hooks +# ------------------------------------------------------------------------- + +def pytest_ignore_collect_path(path, config): + """ return true value to prevent considering this path for collection. + This hook is consulted for all files and directories prior to considering + collection hooks. + """ +pytest_ignore_collect_path.firstresult = True + +def pytest_collect_directory(path, parent): + """ return Collection node or None for the given path. """ +pytest_collect_directory.firstresult = True + +def pytest_collect_file(path, parent): + """ return Collection node or None for the given path. """ + +def pytest_collectstart(collector): + """ collector starts collecting. """ + +def pytest_collectreport(report): + """ collector finished collecting. """ + +def pytest_deselected(items): + """ called for test items deselected by keyword. """ + +def pytest_make_collect_report(collector): + """ perform a collection and return a collection. """ +pytest_make_collect_report.firstresult = True + +# XXX rename to item_collected()? meaning in distribution context? +def pytest_itemstart(item, node=None): + """ test item gets collected. """ + +# ------------------------------------------------------------------------- +# Python test function related hooks +# ------------------------------------------------------------------------- + +def pytest_pycollect_makemodule(path, parent): + """ return a Module collector or None for the given path. + This hook will be called for each matching test module path. + The pytest_collect_file hook needs to be used if you want to + create test modules for files that do not match as a test module. + """ +pytest_pycollect_makemodule.firstresult = True + +def pytest_pycollect_makeitem(collector, name, obj): + """ return custom item/collector for a python object in a module, or None. """ +pytest_pycollect_makeitem.firstresult = True + +def pytest_pyfunc_call(pyfuncitem): + """ call underlying test function. """ +pytest_pyfunc_call.firstresult = True + +def pytest_generate_tests(metafunc): + """ generate (multiple) parametrized calls to a test function.""" + +# ------------------------------------------------------------------------- +# generic runtest related hooks +# ------------------------------------------------------------------------- + +def pytest_runtest_protocol(item): + """ implement fixture, run and report about the given test item. """ +pytest_runtest_protocol.firstresult = True + +def pytest_runtest_setup(item): + """ called before pytest_runtest_call(). """ + +def pytest_runtest_call(item): + """ execute test item. """ + +def pytest_runtest_teardown(item): + """ called after pytest_runtest_call(). """ + +def pytest_runtest_makereport(item, call): + """ make a test report for the given item and call outcome. """ +pytest_runtest_makereport.firstresult = True + +def pytest_runtest_logreport(report): + """ process item test report. """ + +# special handling for final teardown - somewhat internal for now +def pytest__teardown_final(session): + """ called before test session finishes. """ +pytest__teardown_final.firstresult = True + +def pytest__teardown_final_logerror(report): + """ called if runtest_teardown_final failed. """ + +# ------------------------------------------------------------------------- +# test session related hooks +# ------------------------------------------------------------------------- + +def pytest_sessionstart(session): + """ before session.main() is called. """ + +def pytest_sessionfinish(session, exitstatus): + """ whole test run finishes. """ + +# ------------------------------------------------------------------------- +# hooks for influencing reporting (invoked from pytest_terminal) +# ------------------------------------------------------------------------- + +def pytest_report_header(config): + """ return a string to be displayed as header info for terminal reporting.""" + +def pytest_report_teststatus(report): + """ return result-category, shortletter and verbose word for reporting.""" +pytest_report_teststatus.firstresult = True + +def pytest_terminal_summary(terminalreporter): + """ add additional section in terminal summary reporting. """ + +def pytest_report_iteminfo(item): + """ return (fspath, lineno, name) for the item. + the information is used for result display and to sort tests + """ +pytest_report_iteminfo.firstresult = True + +# ------------------------------------------------------------------------- +# doctest hooks +# ------------------------------------------------------------------------- + +def pytest_doctest_prepare_content(content): + """ return processed content for a given doctest""" +pytest_doctest_prepare_content.firstresult = True + + +# ------------------------------------------------------------------------- +# error handling and internal debugging hooks +# ------------------------------------------------------------------------- + +def pytest_plugin_registered(plugin, manager): + """ a new py lib plugin got registered. """ + +def pytest_plugin_unregistered(plugin): + """ a py lib plugin got unregistered. """ + +def pytest_internalerror(excrepr): + """ called for internal errors. """ + +def pytest_keyboard_interrupt(excinfo): + """ called for keyboard interrupt. """ + +def pytest_trace(category, msg): + """ called for debug info. """ Added: pypy/branch/py12/py/_plugin/pytest__pytest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest__pytest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,100 @@ +import py + +from py._test.pluginmanager import HookRelay + +def pytest_funcarg___pytest(request): + return PytestArg(request) + +class PytestArg: + def __init__(self, request): + self.request = request + + def gethookrecorder(self, hook): + hookrecorder = HookRecorder(hook._registry) + hookrecorder.start_recording(hook._hookspecs) + self.request.addfinalizer(hookrecorder.finish_recording) + return hookrecorder + +class ParsedCall: + def __init__(self, name, locals): + assert '_name' not in locals + self.__dict__.update(locals) + self.__dict__.pop('self') + self._name = name + + def __repr__(self): + d = self.__dict__.copy() + del d['_name'] + return "" %(self._name, d) + +class HookRecorder: + def __init__(self, registry): + self._registry = registry + self.calls = [] + self._recorders = {} + + def start_recording(self, hookspecs): + if not isinstance(hookspecs, (list, tuple)): + hookspecs = [hookspecs] + for hookspec in hookspecs: + assert hookspec not in self._recorders + class RecordCalls: + _recorder = self + for name, method in vars(hookspec).items(): + if name[0] != "_": + setattr(RecordCalls, name, self._makecallparser(method)) + recorder = RecordCalls() + self._recorders[hookspec] = recorder + self._registry.register(recorder) + self.hook = HookRelay(hookspecs, registry=self._registry) + + def finish_recording(self): + for recorder in self._recorders.values(): + self._registry.unregister(recorder) + self._recorders.clear() + + def _makecallparser(self, method): + name = method.__name__ + args, varargs, varkw, default = py.std.inspect.getargspec(method) + if not args or args[0] != "self": + args.insert(0, 'self') + fspec = py.std.inspect.formatargspec(args, varargs, varkw, default) + # we use exec because we want to have early type + # errors on wrong input arguments, using + # *args/**kwargs delays this and gives errors + # elsewhere + exec (py.code.compile(""" + def %(name)s%(fspec)s: + self._recorder.calls.append( + ParsedCall(%(name)r, locals())) + """ % locals())) + return locals()[name] + + def getcalls(self, names): + if isinstance(names, str): + names = names.split() + for name in names: + for cls in self._recorders: + if name in vars(cls): + break + else: + raise ValueError("callname %r not found in %r" %( + name, self._recorders.keys())) + l = [] + for call in self.calls: + if call._name in names: + l.append(call) + return l + + def popcall(self, name): + for i, call in enumerate(self.calls): + if call._name == name: + del self.calls[i] + return call + raise ValueError("could not find call %r" %(name, )) + + def getcall(self, name): + l = self.getcalls(name) + assert len(l) == 1, (name, l) + return l[0] + Added: pypy/branch/py12/py/_plugin/pytest_assertion.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_assertion.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,28 @@ +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group._addoption('--no-assert', action="store_true", default=False, + dest="noassert", + help="disable python assert expression reinterpretation."), + +def pytest_configure(config): + if not config.getvalue("noassert") and not config.getvalue("nomagic"): + warn_about_missing_assertion() + config._oldassertion = py.builtin.builtins.AssertionError + py.builtin.builtins.AssertionError = py.code._AssertionError + +def pytest_unconfigure(config): + if hasattr(config, '_oldassertion'): + py.builtin.builtins.AssertionError = config._oldassertion + del config._oldassertion + +def warn_about_missing_assertion(): + try: + assert False + except AssertionError: + pass + else: + py.std.warnings.warn("Assertions are turned off!" + " (are you using python -O?)") Added: pypy/branch/py12/py/_plugin/pytest_capture.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_capture.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,281 @@ +""" +configurable per-test stdout/stderr capturing mechanisms. + +This plugin captures stdout/stderr output for each test separately. +In case of test failures this captured output is shown grouped +togtther with the test. + +The plugin also provides test function arguments that help to +assert stdout/stderr output from within your tests, see the +`funcarg example`_. + + +Capturing of input/output streams during tests +--------------------------------------------------- + +By default ``sys.stdout`` and ``sys.stderr`` are substituted with +temporary streams during the execution of tests and setup/teardown code. +During the whole testing process it will re-use the same temporary +streams allowing to play well with the logging module which easily +takes ownership on these streams. + +Also, 'sys.stdin' is substituted with a file-like "null" object that +does not return any values. This is to immediately error out +on tests that wait on reading something from stdin. + +You can influence output capturing mechanisms from the command line:: + + py.test -s # disable all capturing + py.test --capture=sys # replace sys.stdout/stderr with in-mem files + py.test --capture=fd # point filedescriptors 1 and 2 to temp file + +If you set capturing values in a conftest file like this:: + + # conftest.py + option_capture = 'fd' + +then all tests in that directory will execute with "fd" style capturing. + +sys-level capturing +------------------------------------------ + +Capturing on 'sys' level means that ``sys.stdout`` and ``sys.stderr`` +will be replaced with in-memory files (``py.io.TextIO`` to be precise) +that capture writes and decode non-unicode strings to a unicode object +(using a default, usually, UTF-8, encoding). + +FD-level capturing and subprocesses +------------------------------------------ + +The ``fd`` based method means that writes going to system level files +based on the standard file descriptors will be captured, for example +writes such as ``os.write(1, 'hello')`` will be captured properly. +Capturing on fd-level will include output generated from +any subprocesses created during a test. + +.. _`funcarg example`: + +Example Usage of the capturing Function arguments +--------------------------------------------------- + +You can use the `capsys funcarg`_ and `capfd funcarg`_ to +capture writes to stdout and stderr streams. Using the +funcargs frees your test from having to care about setting/resetting +the old streams and also interacts well with py.test's own +per-test capturing. Here is an example test function: + +.. sourcecode:: python + + def test_myoutput(capsys): + print ("hello") + sys.stderr.write("world\\n") + out, err = capsys.readouterr() + assert out == "hello\\n" + assert err == "world\\n" + print "next" + out, err = capsys.readouterr() + assert out == "next\\n" + +The ``readouterr()`` call snapshots the output so far - +and capturing will be continued. After the test +function finishes the original streams will +be restored. If you want to capture on +the filedescriptor level you can use the ``capfd`` function +argument which offers the same interface. +""" + +import py +import os + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--capture', action="store", default=None, + metavar="method", type="choice", choices=['fd', 'sys', 'no'], + help="per-test capturing method: one of fd (default)|sys|no.") + group._addoption('-s', action="store_const", const="no", dest="capture", + help="shortcut for --capture=no.") + +def addouterr(rep, outerr): + repr = getattr(rep, 'longrepr', None) + if not hasattr(repr, 'addsection'): + return + for secname, content in zip(["out", "err"], outerr): + if content: + repr.addsection("Captured std%s" % secname, content.rstrip()) + +def pytest_configure(config): + config.pluginmanager.register(CaptureManager(), 'capturemanager') + +class CaptureManager: + def __init__(self): + self._method2capture = {} + + def _maketempfile(self): + f = py.std.tempfile.TemporaryFile() + newf = py.io.dupfile(f, encoding="UTF-8") + return newf + + def _makestringio(self): + return py.io.TextIO() + + def _startcapture(self, method): + if method == "fd": + return py.io.StdCaptureFD( + out=self._maketempfile(), err=self._maketempfile() + ) + elif method == "sys": + return py.io.StdCapture( + out=self._makestringio(), err=self._makestringio() + ) + else: + raise ValueError("unknown capturing method: %r" % method) + + def _getmethod(self, config, fspath): + if config.option.capture: + method = config.option.capture + else: + try: + method = config._conftest.rget("option_capture", path=fspath) + except KeyError: + method = "fd" + if method == "fd" and not hasattr(os, 'dup'): # e.g. jython + method = "sys" + return method + + def resumecapture_item(self, item): + method = self._getmethod(item.config, item.fspath) + if not hasattr(item, 'outerr'): + item.outerr = ('', '') # we accumulate outerr on the item + return self.resumecapture(method) + + def resumecapture(self, method): + if hasattr(self, '_capturing'): + raise ValueError("cannot resume, already capturing with %r" % + (self._capturing,)) + if method != "no": + cap = self._method2capture.get(method) + if cap is None: + cap = self._startcapture(method) + self._method2capture[method] = cap + else: + cap.resume() + self._capturing = method + + def suspendcapture(self, item=None): + self.deactivate_funcargs() + if hasattr(self, '_capturing'): + method = self._capturing + if method != "no": + cap = self._method2capture[method] + outerr = cap.suspend() + else: + outerr = "", "" + del self._capturing + if item: + outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + return outerr + return "", "" + + def activate_funcargs(self, pyfuncitem): + if not hasattr(pyfuncitem, 'funcargs'): + return + assert not hasattr(self, '_capturing_funcargs') + l = [] + for name, obj in pyfuncitem.funcargs.items(): + if name == 'capfd' and not hasattr(os, 'dup'): + py.test.skip("capfd funcarg needs os.dup") + if name in ('capsys', 'capfd'): + obj._start() + l.append(obj) + if l: + self._capturing_funcargs = l + + def deactivate_funcargs(self): + if hasattr(self, '_capturing_funcargs'): + for capfuncarg in self._capturing_funcargs: + capfuncarg._finalize() + del self._capturing_funcargs + + def pytest_make_collect_report(self, __multicall__, collector): + method = self._getmethod(collector.config, collector.fspath) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + addouterr(rep, outerr) + return rep + + def pytest_runtest_setup(self, item): + self.resumecapture_item(item) + + def pytest_runtest_call(self, item): + self.resumecapture_item(item) + self.activate_funcargs(item) + + def pytest_runtest_teardown(self, item): + self.resumecapture_item(item) + + def pytest__teardown_final(self, __multicall__, session): + method = self._getmethod(session.config, None) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + if rep: + addouterr(rep, outerr) + return rep + + def pytest_keyboard_interrupt(self, excinfo): + if hasattr(self, '_capturing'): + self.suspendcapture() + + def pytest_runtest_makereport(self, __multicall__, item, call): + self.deactivate_funcargs() + rep = __multicall__.execute() + outerr = self.suspendcapture(item) + if not rep.passed: + addouterr(rep, outerr) + if not rep.passed or rep.when == "teardown": + outerr = ('', '') + item.outerr = outerr + return rep + +def pytest_funcarg__capsys(request): + """captures writes to sys.stdout/sys.stderr and makes + them available successively via a ``capsys.readouterr()`` method + which returns a ``(out, err)`` tuple of captured snapshot strings. + """ + return CaptureFuncarg(request, py.io.StdCapture) + +def pytest_funcarg__capfd(request): + """captures writes to file descriptors 1 and 2 and makes + snapshotted ``(out, err)`` string tuples available + via the ``capsys.readouterr()`` method. If the underlying + platform does not have ``os.dup`` (e.g. Jython) tests using + this funcarg will automatically skip. + """ + return CaptureFuncarg(request, py.io.StdCaptureFD) + + +class CaptureFuncarg: + def __init__(self, request, captureclass): + self._cclass = captureclass + #request.addfinalizer(self._finalize) + + def _start(self): + self.capture = self._cclass() + + def _finalize(self): + if hasattr(self, 'capture'): + self.capture.reset() + del self.capture + + def readouterr(self): + return self.capture.readouterr() + + def close(self): + self.capture.reset() + del self.capture + Added: pypy/branch/py12/py/_plugin/pytest_default.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_default.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,125 @@ +""" default hooks and general py.test options. """ + +import sys +import py + +def pytest_pyfunc_call(__multicall__, pyfuncitem): + if not __multicall__.execute(): + testfunction = pyfuncitem.obj + if pyfuncitem._isyieldedfunction(): + testfunction(*pyfuncitem._args) + else: + funcargs = pyfuncitem.funcargs + testfunction(**funcargs) + +def pytest_collect_file(path, parent): + ext = path.ext + pb = path.purebasename + if pb.startswith("test_") or pb.endswith("_test") or \ + path in parent.config._argfspaths: + if ext == ".py": + return parent.ihook.pytest_pycollect_makemodule( + path=path, parent=parent) + +def pytest_pycollect_makemodule(path, parent): + return parent.Module(path, parent) + +def pytest_funcarg__pytestconfig(request): + """ the pytest config object with access to command line opts.""" + return request.config + +def pytest_ignore_collect_path(path, config): + ignore_paths = config.getconftest_pathlist("collect_ignore", path=path) + ignore_paths = ignore_paths or [] + excludeopt = config.getvalue("ignore") + if excludeopt: + ignore_paths.extend([py.path.local(x) for x in excludeopt]) + return path in ignore_paths + # XXX more refined would be: + if ignore_paths: + for p in ignore_paths: + if path == p or path.relto(p): + return True + + +def pytest_collect_directory(path, parent): + # XXX reconsider the following comment + # not use parent.Directory here as we generally + # want dir/conftest.py to be able to + # define Directory(dir) already + if not parent.recfilter(path): # by default special ".cvs", ... + # check if cmdline specified this dir or a subdir directly + for arg in parent.config._argfspaths: + if path == arg or arg.relto(path): + break + else: + return + Directory = parent.config._getcollectclass('Directory', path) + return Directory(path, parent=parent) + +def pytest_report_iteminfo(item): + return item.reportinfo() + +def pytest_addoption(parser): + group = parser.getgroup("general", "running and selection options") + group._addoption('-x', '--exitfirst', + action="store_true", dest="exitfirst", default=False, + help="exit instantly on first error or failed test."), + group._addoption('-k', + action="store", dest="keyword", default='', + help="only run test items matching the given " + "space separated keywords. precede a keyword with '-' to negate. " + "Terminate the expression with ':' to treat a match as a signal " + "to run all subsequent tests. ") + + group = parser.getgroup("collect", "collection") + group.addoption('--collectonly', + action="store_true", dest="collectonly", + help="only collect tests, don't execute them."), + group.addoption("--ignore", action="append", metavar="path", + help="ignore path during collection (multi-allowed).") + group.addoption('--confcutdir', dest="confcutdir", default=None, + metavar="dir", + help="only load conftest.py's relative to specified dir.") + + group = parser.getgroup("debugconfig", + "test process debugging and configuration") + group.addoption('--basetemp', dest="basetemp", default=None, metavar="dir", + help="base temporary directory for this test run.") + +def pytest_configure(config): + setsession(config) + +def setsession(config): + val = config.getvalue + if val("collectonly"): + from py._test.session import Session + config.setsessionclass(Session) + +# pycollect related hooks and code, should move to pytest_pycollect.py + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + res = __multicall__.execute() + if res is not None: + return res + if collector._istestclasscandidate(name, obj): + res = collector._deprecated_join(name) + if res is not None: + return res + return collector.Class(name, parent=collector) + elif collector.funcnamefilter(name) and hasattr(obj, '__call__'): + res = collector._deprecated_join(name) + if res is not None: + return res + if is_generator(obj): + # XXX deprecation warning + return collector.Generator(name, parent=collector) + else: + return collector._genfunctions(name, obj) + +def is_generator(func): + try: + return py.code.getrawcode(func).co_flags & 32 # generator function + except AttributeError: # builtin functions have no bytecode + # assume them to not be generators + return False Added: pypy/branch/py12/py/_plugin/pytest_doctest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_doctest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,100 @@ +""" +collect and execute doctests from modules and test files. + +Usage +------------- + +By default all files matching the ``test*.txt`` pattern will +be run through the python standard ``doctest`` module. Issue:: + + py.test --doctest-glob='*.rst' + +to change the pattern. Additionally you can trigger running of +tests in all python modules (including regular python test modules):: + + py.test --doctest-modules + +You can also make these changes permanent in your project by +putting them into a conftest.py file like this:: + + # content of conftest.py + option_doctestmodules = True + option_doctestglob = "*.rst" +""" + +import py +from py._code.code import TerminalRepr, ReprFileLocation +import doctest + +def pytest_addoption(parser): + group = parser.getgroup("collect") + group.addoption("--doctest-modules", + action="store_true", default=False, + help="run doctests in all .py modules", + dest="doctestmodules") + group.addoption("--doctest-glob", + action="store", default="test*.txt", metavar="pat", + help="doctests file matching pattern, default: test*.txt", + dest="doctestglob") + +def pytest_collect_file(path, parent): + config = parent.config + if path.ext == ".py": + if config.getvalue("doctestmodules"): + return DoctestModule(path, parent) + elif path.check(fnmatch=config.getvalue("doctestglob")): + return DoctestTextfile(path, parent) + +class ReprFailDoctest(TerminalRepr): + def __init__(self, reprlocation, lines): + self.reprlocation = reprlocation + self.lines = lines + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + self.reprlocation.toterminal(tw) + +class DoctestItem(py.test.collect.Item): + def __init__(self, path, parent): + name = self.__class__.__name__ + ":" + path.basename + super(DoctestItem, self).__init__(name=name, parent=parent) + self.fspath = path + + def repr_failure(self, excinfo): + if excinfo.errisinstance(doctest.DocTestFailure): + doctestfailure = excinfo.value + example = doctestfailure.example + test = doctestfailure.test + filename = test.filename + lineno = test.lineno + example.lineno + 1 + message = excinfo.type.__name__ + reprlocation = ReprFileLocation(filename, lineno, message) + checker = doctest.OutputChecker() + REPORT_UDIFF = doctest.REPORT_UDIFF + filelines = py.path.local(filename).readlines(cr=0) + i = max(test.lineno, max(0, lineno - 10)) # XXX? + lines = [] + for line in filelines[i:lineno]: + lines.append("%03d %s" % (i+1, line)) + i += 1 + lines += checker.output_difference(example, + doctestfailure.got, REPORT_UDIFF).split("\n") + return ReprFailDoctest(reprlocation, lines) + elif excinfo.errisinstance(doctest.UnexpectedException): + excinfo = py.code.ExceptionInfo(excinfo.value.exc_info) + return super(DoctestItem, self).repr_failure(excinfo) + else: + return super(DoctestItem, self).repr_failure(excinfo) + +class DoctestTextfile(DoctestItem): + def runtest(self): + if not self._deprecated_testexecution(): + failed, tot = doctest.testfile( + str(self.fspath), module_relative=False, + raise_on_error=True, verbose=0) + +class DoctestModule(DoctestItem): + def runtest(self): + module = self.fspath.pyimport() + failed, tot = doctest.testmod( + module, raise_on_error=True, verbose=0) Added: pypy/branch/py12/py/_plugin/pytest_genscript.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_genscript.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,69 @@ +#! /usr/bin/env python +""" +generate standalone test script to be distributed along with an application. +""" + +import os +import zlib +import base64 +import sys +try: + import pickle +except Importerror: + import cPickle as pickle + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group.addoption("--genscript", action="store", default=None, + dest="genscript", metavar="path", + help="create standalone py.test script at given target path.") + +def pytest_configure(config): + genscript = config.getvalue("genscript") + if genscript: + import py + mydir = py.path.local(__file__).dirpath() + infile = mydir.join("standalonetemplate.py") + pybasedir = py.path.local(py.__file__).dirpath().dirpath() + genscript = py.path.local(genscript) + main(pybasedir, outfile=genscript, infile=infile) + raise SystemExit(0) + +def main(pybasedir, outfile, infile): + outfile = str(outfile) + infile = str(infile) + assert os.path.isabs(outfile) + os.chdir(str(pybasedir)) + files = [] + for dirpath, dirnames, filenames in os.walk("py"): + for f in filenames: + if not f.endswith(".py"): + continue + + fn = os.path.join(dirpath, f) + files.append(fn) + + name2src = {} + for f in files: + k = f.replace(os.sep, ".")[:-3] + name2src[k] = open(f, "r").read() + + data = pickle.dumps(name2src, 2) + data = zlib.compress(data, 9) + data = base64.encodestring(data) + data = data.decode("ascii") + + exe = open(infile, "r").read() + exe = exe.replace("@SOURCES@", data) + + open(outfile, "w").write(exe) + os.chmod(outfile, 493) # 0755 + sys.stdout.write("generated standalone py.test at %r, have fun!\n" % outfile) + +if __name__=="__main__": + dn = os.path.dirname + here = os.path.abspath(dn(__file__)) # py/plugin/ + pybasedir = dn(dn(here)) + outfile = os.path.join(os.getcwd(), "py.test-standalone") + infile = os.path.join(here, 'standalonetemplate.py') + main(pybasedir, outfile, infile) Added: pypy/branch/py12/py/_plugin/pytest_helpconfig.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_helpconfig.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,164 @@ +""" provide version info, conftest/environment config names. +""" +import py +import inspect, sys + +def pytest_addoption(parser): + group = parser.getgroup('debugconfig') + group.addoption('--version', action="store_true", + help="display py lib version and import information.") + group._addoption('-p', action="append", dest="plugins", default = [], + metavar="name", + help="early-load given plugin (multi-allowed).") + group.addoption('--traceconfig', + action="store_true", dest="traceconfig", default=False, + help="trace considerations of conftest.py files."), + group._addoption('--nomagic', + action="store_true", dest="nomagic", default=False, + help="don't reinterpret asserts, no traceback cutting. ") + group.addoption('--debug', + action="store_true", dest="debug", default=False, + help="generate and show internal debugging information.") + group.addoption("--help-config", action="store_true", dest="helpconfig", + help="show available conftest.py and ENV-variable names.") + + +def pytest_configure(__multicall__, config): + if config.option.version: + p = py.path.local(py.__file__).dirpath() + sys.stderr.write("This is py.test version %s, imported from %s\n" % + (py.__version__, p)) + sys.exit(0) + if not config.option.helpconfig: + return + __multicall__.execute() + options = [] + for group in config._parser._groups: + options.extend(group.options) + widths = [0] * 10 + tw = py.io.TerminalWriter() + tw.sep("-") + tw.line("%-13s | %-18s | %-25s | %s" %( + "cmdline name", "conftest.py name", "ENV-variable name", "help")) + tw.sep("-") + + options = [opt for opt in options if opt._long_opts] + options.sort(key=lambda x: x._long_opts) + for opt in options: + if not opt._long_opts: + continue + optstrings = list(opt._long_opts) # + list(opt._short_opts) + optstrings = filter(None, optstrings) + optstring = "|".join(optstrings) + line = "%-13s | %-18s | %-25s | %s" %( + optstring, + "option_%s" % opt.dest, + "PYTEST_OPTION_%s" % opt.dest.upper(), + opt.help and opt.help or "", + ) + tw.line(line[:tw.fullwidth]) + for name, help in conftest_options: + line = "%-13s | %-18s | %-25s | %s" %( + "", + name, + "", + help, + ) + tw.line(line[:tw.fullwidth]) + + tw.sep("-") + sys.exit(0) + +conftest_options = ( + ('pytest_plugins', 'list of plugin names to load'), + ('collect_ignore', '(relative) paths ignored during collection'), + ('rsyncdirs', 'to-be-rsynced directories for dist-testing'), +) + +def pytest_report_header(config): + lines = [] + if config.option.debug or config.option.traceconfig: + lines.append("using py lib: %s" % (py.path.local(py.__file__).dirpath())) + if config.option.traceconfig: + lines.append("active plugins:") + plugins = [] + items = config.pluginmanager._name2plugin.items() + for name, plugin in items: + lines.append(" %-20s: %s" %(name, repr(plugin))) + return lines + + +# ===================================================== +# validate plugin syntax and hooks +# ===================================================== + +def pytest_plugin_registered(manager, plugin): + methods = collectattr(plugin) + hooks = {} + for hookspec in manager.hook._hookspecs: + hooks.update(collectattr(hookspec)) + + stringio = py.io.TextIO() + def Print(*args): + if args: + stringio.write(" ".join(map(str, args))) + stringio.write("\n") + + fail = False + while methods: + name, method = methods.popitem() + #print "checking", name + if isgenerichook(name): + continue + if name not in hooks: + if not getattr(method, 'optionalhook', False): + Print("found unknown hook:", name) + fail = True + else: + #print "checking", method + method_args = getargs(method) + #print "method_args", method_args + if '__multicall__' in method_args: + method_args.remove('__multicall__') + hook = hooks[name] + hookargs = getargs(hook) + for arg in method_args: + if arg not in hookargs: + Print("argument %r not available" %(arg, )) + Print("actual definition: %s" %(formatdef(method))) + Print("available hook arguments: %s" % + ", ".join(hookargs)) + fail = True + break + #if not fail: + # print "matching hook:", formatdef(method) + if fail: + name = getattr(plugin, '__name__', plugin) + raise PluginValidationError("%s:\n%s" %(name, stringio.getvalue())) + +class PluginValidationError(Exception): + """ plugin failed validation. """ + +def isgenerichook(name): + return name == "pytest_plugins" or \ + name.startswith("pytest_funcarg__") + +def getargs(func): + args = inspect.getargs(py.code.getrawcode(func))[0] + startindex = inspect.ismethod(func) and 1 or 0 + return args[startindex:] + +def collectattr(obj, prefixes=("pytest_",)): + methods = {} + for apiname in dir(obj): + for prefix in prefixes: + if apiname.startswith(prefix): + methods[apiname] = getattr(obj, apiname) + return methods + +def formatdef(func): + return "%s%s" %( + func.__name__, + inspect.formatargspec(*inspect.getargspec(func)) + ) + Added: pypy/branch/py12/py/_plugin/pytest_hooklog.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_hooklog.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,33 @@ +""" log invocations of extension hooks to a file. """ +import py + +def pytest_addoption(parser): + parser.addoption("--hooklog", dest="hooklog", default=None, + help="write hook calls to the given file.") + +def pytest_configure(config): + hooklog = config.getvalue("hooklog") + if hooklog: + config._hooklogfile = open(hooklog, 'w') + config._hooklog_oldperformcall = config.hook._performcall + config.hook._performcall = (lambda name, multicall: + logged_call(name=name, multicall=multicall, config=config)) + +def logged_call(name, multicall, config): + f = config._hooklogfile + f.write("%s(**%s)\n" % (name, multicall.kwargs)) + try: + res = config._hooklog_oldperformcall(name=name, multicall=multicall) + except: + f.write("-> exception") + raise + f.write("-> %r" % (res,)) + return res + +def pytest_unconfigure(config): + try: + del config.hook.__dict__['_performcall'] + except KeyError: + pass + else: + config._hooklogfile.close() Added: pypy/branch/py12/py/_plugin/pytest_junitxml.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_junitxml.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,161 @@ +""" + logging of test results in JUnit-XML format, for use with Hudson + and build integration servers. Based on initial code from Ross Lawley. +""" + +import py +import time + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting") + group.addoption('--junitxml', action="store", dest="xmlpath", + metavar="path", default=None, + help="create junit-xml style report file at given path.") + +def pytest_configure(config): + xmlpath = config.option.xmlpath + if xmlpath: + config._xml = LogXML(xmlpath) + config.pluginmanager.register(config._xml) + +def pytest_unconfigure(config): + xml = getattr(config, '_xml', None) + if xml: + del config._xml + config.pluginmanager.unregister(xml) + +class LogXML(object): + def __init__(self, logfile): + self.logfile = logfile + self.test_logs = [] + self.passed = self.skipped = 0 + self.failed = self.errors = 0 + self._durations = {} + + def _opentestcase(self, report): + node = report.item + d = {'time': self._durations.pop(report.item, "0")} + names = [x.replace(".py", "") for x in node.listnames() if x != "()"] + d['classname'] = ".".join(names[:-1]) + d['name'] = names[-1] + attrs = ['%s="%s"' % item for item in sorted(d.items())] + self.test_logs.append("\n" % " ".join(attrs)) + + def _closetestcase(self): + self.test_logs.append("") + + def appendlog(self, fmt, *args): + args = tuple([py.xml.escape(arg) for arg in args]) + self.test_logs.append(fmt % args) + + def append_pass(self, report): + self.passed += 1 + self._opentestcase(report) + self._closetestcase() + + def append_failure(self, report): + self._opentestcase(report) + #msg = str(report.longrepr.reprtraceback.extraline) + self.appendlog('%s', + report.longrepr) + self._closetestcase() + self.failed += 1 + + def _opentestcase_collectfailure(self, report): + node = report.collector + d = {'time': '???'} + names = [x.replace(".py", "") for x in node.listnames() if x != "()"] + d['classname'] = ".".join(names[:-1]) + d['name'] = names[-1] + attrs = ['%s="%s"' % item for item in sorted(d.items())] + self.test_logs.append("\n" % " ".join(attrs)) + + def append_collect_failure(self, report): + self._opentestcase_collectfailure(report) + #msg = str(report.longrepr.reprtraceback.extraline) + self.appendlog('%s', + report.longrepr) + self._closetestcase() + self.errors += 1 + + def append_collect_skipped(self, report): + self._opentestcase_collectfailure(report) + #msg = str(report.longrepr.reprtraceback.extraline) + self.appendlog('%s', + report.longrepr) + self._closetestcase() + self.skipped += 1 + + def append_error(self, report): + self._opentestcase(report) + self.appendlog('%s', + report.longrepr) + self._closetestcase() + self.errors += 1 + + def append_skipped(self, report): + self._opentestcase(report) + self.appendlog("") + self._closetestcase() + self.skipped += 1 + + def pytest_runtest_logreport(self, report): + if report.passed: + self.append_pass(report) + elif report.failed: + if report.when != "call": + self.append_error(report) + else: + self.append_failure(report) + elif report.skipped: + self.append_skipped(report) + + def pytest_runtest_call(self, item, __multicall__): + start = time.time() + try: + return __multicall__.execute() + finally: + self._durations[item] = time.time() - start + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.append_collect_failure(report) + else: + self.append_collect_skipped(report) + + def pytest_internalerror(self, excrepr): + self.errors += 1 + data = py.xml.escape(excrepr) + self.test_logs.append( + '\n' + ' ' + '%s' % data) + + def pytest_sessionstart(self, session): + self.suite_start_time = time.time() + + def pytest_sessionfinish(self, session, exitstatus, __multicall__): + if py.std.sys.version_info[0] < 3: + logfile = py.std.codecs.open(self.logfile, 'w', encoding='utf-8') + else: + logfile = open(self.logfile, 'w', encoding='utf-8') + + suite_stop_time = time.time() + suite_time_delta = suite_stop_time - self.suite_start_time + numtests = self.passed + self.failed + logfile.write('') + logfile.write('') + logfile.writelines(self.test_logs) + logfile.write('') + logfile.close() + tw = session.config.pluginmanager.getplugin("terminalreporter")._tw + tw.line() + tw.sep("-", "generated xml file: %s" %(self.logfile)) Added: pypy/branch/py12/py/_plugin/pytest_mark.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_mark.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,152 @@ +""" +generic mechanism for marking python functions. + +By using the ``py.test.mark`` helper you can instantiate +decorators that will set named meta data on test functions. + +Marking a single function +---------------------------------------------------- + +You can "mark" a test function with meta data like this:: + + @py.test.mark.webtest + def test_send_http(): + ... + +This will set a "Marker" instance as a function attribute named "webtest". +You can also specify parametrized meta data like this:: + + @py.test.mark.webtest(firefox=30) + def test_receive(): + ... + +The named marker can be accessed like this later:: + + test_receive.webtest.kwargs['firefox'] == 30 + +In addition to set key-value pairs you can also use positional arguments:: + + @py.test.mark.webtest("triangular") + def test_receive(): + ... + +and later access it with ``test_receive.webtest.args[0] == 'triangular``. + +.. _`scoped-marking`: + +Marking classes or modules +---------------------------------------------------- + +To mark all methods of a class set a ``pytestmark`` attribute like this:: + + import py + + class TestClass: + pytestmark = py.test.mark.webtest + +You can re-use the same markers that you would use for decorating +a function - in fact this marker decorator will be applied +to all test methods of the class. + +You can also set a module level marker:: + + import py + pytestmark = py.test.mark.webtest + +in which case then the marker decorator will be applied to all functions and +methods defined in the module. + +The order in which marker functions are called is this:: + + per-function (upon import of module already) + per-class + per-module + +Later called markers may overwrite previous key-value settings. +Positional arguments are all appended to the same 'args' list +of the Marker object. + +Using "-k MARKNAME" to select tests +---------------------------------------------------- + +You can use the ``-k`` command line option to select +tests:: + + py.test -k webtest # will only run tests marked as webtest + +""" +import py + +def pytest_namespace(): + return {'mark': MarkGenerator()} + +class MarkGenerator: + """ non-underscore attributes of this object can be used as decorators for + marking test functions. Example: @py.test.mark.slowtest in front of a + function will set the 'slowtest' marker object on it. """ + def __getattr__(self, name): + if name[0] == "_": + raise AttributeError(name) + return MarkDecorator(name) + +class MarkDecorator: + """ decorator for setting function attributes. """ + def __init__(self, name): + self.markname = name + self.kwargs = {} + self.args = [] + + def __repr__(self): + d = self.__dict__.copy() + name = d.pop('markname') + return "" %(name, d) + + def __call__(self, *args, **kwargs): + """ if passed a single callable argument: decorate it with mark info. + otherwise add *args/**kwargs in-place to mark information. """ + if args: + if len(args) == 1 and hasattr(args[0], '__call__'): + func = args[0] + holder = getattr(func, self.markname, None) + if holder is None: + holder = MarkInfo(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) + return func + else: + self.args.extend(args) + self.kwargs.update(kwargs) + return self + +class MarkInfo: + def __init__(self, name, args, kwargs): + self._name = name + self.args = args + self.kwargs = kwargs + + def __getattr__(self, name): + if name[0] != '_' and name in self.kwargs: + py.log._apiwarn("1.1", "use .kwargs attribute to access key-values") + return self.kwargs[name] + raise AttributeError(name) + + def __repr__(self): + return "" % ( + self._name, self.args, self.kwargs) + + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + item = __multicall__.execute() + if isinstance(item, py.test.collect.Function): + cls = collector.getparent(py.test.collect.Class) + mod = collector.getparent(py.test.collect.Module) + func = item.obj + func = getattr(func, '__func__', func) # py3 + func = getattr(func, 'im_func', func) # py2 + for parent in [x for x in (mod, cls) if x]: + marker = getattr(parent.obj, 'pytestmark', None) + if isinstance(marker, MarkDecorator): + marker(func) + return item Added: pypy/branch/py12/py/_plugin/pytest_monkeypatch.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_monkeypatch.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,141 @@ +""" +safely patch object attributes, dicts and environment variables. + +Usage +---------------- + +Use the `monkeypatch funcarg`_ to tweak your global test environment +for running a particular test. You can safely set/del an attribute, +dictionary item or environment variable by respective methods +on the monkeypatch funcarg. If you want e.g. to set an ENV1 variable +and have os.path.expanduser return a particular directory, you can +write it down like this: + +.. sourcecode:: python + + def test_mytest(monkeypatch): + monkeypatch.setenv('ENV1', 'myval') + monkeypatch.setattr(os.path, 'expanduser', lambda x: '/tmp/xyz') + ... # your test code that uses those patched values implicitely + +After the test function finished all modifications will be undone, +because the ``monkeypatch.undo()`` method is registered as a finalizer. + +``monkeypatch.setattr/delattr/delitem/delenv()`` all +by default raise an Exception if the target does not exist. +Pass ``raising=False`` if you want to skip this check. + +prepending to PATH or other environment variables +--------------------------------------------------------- + +To prepend a value to an already existing environment parameter: + +.. sourcecode:: python + + def test_mypath_finding(monkeypatch): + monkeypatch.setenv('PATH', 'x/y', prepend=":") + # in bash language: export PATH=x/y:$PATH + +calling "undo" finalization explicitely +----------------------------------------- + +At the end of function execution py.test invokes +a teardown hook which undoes all monkeypatch changes. +If you do not want to wait that long you can call +finalization explicitely:: + + monkeypatch.undo() + +This will undo previous changes. This call consumes the +undo stack. Calling it a second time has no effect unless +you start monkeypatching after the undo call. + +.. _`monkeypatch blog post`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ +""" + +import py, os, sys + +def pytest_funcarg__monkeypatch(request): + """The returned ``monkeypatch`` funcarg provides these + helper methods to modify objects, dictionaries or os.environ:: + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=False) + monkeypatch.delenv(name, value, raising=True) + monkeypatch.syspath_prepend(path) + + All modifications will be undone when the requesting + test function finished its execution. The ``raising`` + parameter determines if a KeyError or AttributeError + will be raised if the set/deletion operation has no target. + """ + monkeypatch = MonkeyPatch() + request.addfinalizer(monkeypatch.undo) + return monkeypatch + +notset = object() + +class MonkeyPatch: + def __init__(self): + self._setattr = [] + self._setitem = [] + + def setattr(self, obj, name, value, raising=True): + oldval = getattr(obj, name, notset) + if raising and oldval is notset: + raise AttributeError("%r has no attribute %r" %(obj, name)) + self._setattr.insert(0, (obj, name, oldval)) + setattr(obj, name, value) + + def delattr(self, obj, name, raising=True): + if not hasattr(obj, name): + if raising: + raise AttributeError(name) + else: + self._setattr.insert(0, (obj, name, getattr(obj, name, notset))) + delattr(obj, name) + + def setitem(self, dic, name, value): + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + dic[name] = value + + def delitem(self, dic, name, raising=True): + if name not in dic: + if raising: + raise KeyError(name) + else: + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + del dic[name] + + def setenv(self, name, value, prepend=None): + value = str(value) + if prepend and name in os.environ: + value = value + prepend + os.environ[name] + self.setitem(os.environ, name, value) + + def delenv(self, name, raising=True): + self.delitem(os.environ, name, raising=raising) + + def syspath_prepend(self, path): + if not hasattr(self, '_savesyspath'): + self._savesyspath = sys.path[:] + sys.path.insert(0, str(path)) + + def undo(self): + for obj, name, value in self._setattr: + if value is not notset: + setattr(obj, name, value) + else: + delattr(obj, name) + self._setattr[:] = [] + for dictionary, name, value in self._setitem: + if value is notset: + del dictionary[name] + else: + dictionary[name] = value + self._setitem[:] = [] + if hasattr(self, '_savesyspath'): + sys.path[:] = self._savesyspath Added: pypy/branch/py12/py/_plugin/pytest_nose.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_nose.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,98 @@ +"""nose-compatibility plugin: allow to run nose test suites natively. + +This is an experimental plugin for allowing to run tests written +in 'nosetests style with py.test. + +Usage +------------- + +type:: + + py.test # instead of 'nosetests' + +and you should be able to run nose style tests and at the same +time can make full use of py.test's capabilities. + +Supported nose Idioms +---------------------- + +* setup and teardown at module/class/method level +* SkipTest exceptions and markers +* setup/teardown decorators +* yield-based tests and their setup +* general usage of nose utilities + +Unsupported idioms / issues +---------------------------------- + +- nose-style doctests are not collected and executed correctly, + also fixtures don't work. + +- no nose-configuration is recognized + +If you find other issues or have suggestions please run:: + + py.test --pastebin=all + +and send the resulting URL to a py.test contact channel, +at best to the mailing list. +""" +import py +import inspect +import sys + +def pytest_runtest_makereport(__multicall__, item, call): + SkipTest = getattr(sys.modules.get('nose', None), 'SkipTest', None) + if SkipTest: + if call.excinfo and call.excinfo.errisinstance(SkipTest): + # let's substitute the excinfo with a py.test.skip one + call2 = call.__class__(lambda: py.test.skip(str(call.excinfo.value)), call.when) + call.excinfo = call2.excinfo + +def pytest_report_iteminfo(item): + # nose 0.11.1 uses decorators for "raises" and other helpers. + # for reporting progress by filename we fish for the filename + if isinstance(item, py.test.collect.Function): + obj = item.obj + if hasattr(obj, 'compat_co_firstlineno'): + fn = sys.modules[obj.__module__].__file__ + if fn.endswith(".pyc"): + fn = fn[:-1] + #assert 0 + #fn = inspect.getsourcefile(obj) or inspect.getfile(obj) + lineno = obj.compat_co_firstlineno + return py.path.local(fn), lineno, obj.__module__ + +def pytest_runtest_setup(item): + if isinstance(item, (py.test.collect.Function)): + if isinstance(item.parent, py.test.collect.Generator): + gen = item.parent + if not hasattr(gen, '_nosegensetup'): + call_optional(gen.obj, 'setup') + if isinstance(gen.parent, py.test.collect.Instance): + call_optional(gen.parent.obj, 'setup') + gen._nosegensetup = True + if not call_optional(item.obj, 'setup'): + # call module level setup if there is no object level one + call_optional(item.parent.obj, 'setup') + +def pytest_runtest_teardown(item): + if isinstance(item, py.test.collect.Function): + if not call_optional(item.obj, 'teardown'): + call_optional(item.parent.obj, 'teardown') + #if hasattr(item.parent, '_nosegensetup'): + # #call_optional(item._nosegensetup, 'teardown') + # del item.parent._nosegensetup + +def pytest_make_collect_report(collector): + if isinstance(collector, py.test.collect.Generator): + call_optional(collector.obj, 'setup') + +def call_optional(obj, name): + method = getattr(obj, name, None) + if method: + ismethod = inspect.ismethod(method) + rawcode = py.code.getrawcode(method) + if not rawcode.co_varnames[ismethod:]: + method() + return True Added: pypy/branch/py12/py/_plugin/pytest_pastebin.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_pastebin.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,83 @@ +""" +submit failure or test session information to a pastebin service. + +Usage +---------- + +**Creating a URL for each test failure**:: + + py.test --pastebin=failed + +This will submit test run information to a remote Paste service and +provide a URL for each failure. You may select tests as usual or add +for example ``-x`` if you only want to send one particular failure. + +**Creating a URL for a whole test session log**:: + + py.test --pastebin=all + +Currently only pasting to the http://paste.pocoo.org service is implemented. + +""" +import py, sys + +class url: + base = "http://paste.pocoo.org" + xmlrpc = base + "/xmlrpc/" + show = base + "/show/" + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting") + group._addoption('--pastebin', metavar="mode", + action='store', dest="pastebin", default=None, + type="choice", choices=['failed', 'all'], + help="send failed|all info to Pocoo pastebin service.") + +def pytest_configure(__multicall__, config): + import tempfile + __multicall__.execute() + if config.option.pastebin == "all": + config._pastebinfile = tempfile.TemporaryFile('w+') + tr = config.pluginmanager.getplugin('terminalreporter') + oldwrite = tr._tw.write + def tee_write(s, **kwargs): + oldwrite(s, **kwargs) + config._pastebinfile.write(str(s)) + tr._tw.write = tee_write + +def pytest_unconfigure(config): + if hasattr(config, '_pastebinfile'): + config._pastebinfile.seek(0) + sessionlog = config._pastebinfile.read() + config._pastebinfile.close() + del config._pastebinfile + proxyid = getproxy().newPaste("python", sessionlog) + pastebinurl = "%s%s" % (url.show, proxyid) + sys.stderr.write("pastebin session-log: %s\n" % pastebinurl) + tr = config.pluginmanager.getplugin('terminalreporter') + del tr._tw.__dict__['write'] + +def getproxy(): + return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes + +def pytest_terminal_summary(terminalreporter): + if terminalreporter.config.option.pastebin != "failed": + return + tr = terminalreporter + if 'failed' in tr.stats: + terminalreporter.write_sep("=", "Sending information to Paste Service") + if tr.config.option.debug: + terminalreporter.write_line("xmlrpcurl: %s" %(url.xmlrpc,)) + serverproxy = getproxy() + for rep in terminalreporter.stats.get('failed'): + try: + msg = rep.longrepr.reprtraceback.reprentries[-1].reprfileloc + except AttributeError: + msg = tr._getfailureheadline(rep) + tw = py.io.TerminalWriter(stringio=True) + rep.toterminal(tw) + s = tw.stringio.getvalue() + assert len(s) + proxyid = serverproxy.newPaste("python", s) + pastebinurl = "%s%s" % (url.show, proxyid) + tr.write_line("%s --> %s" %(msg, pastebinurl)) Added: pypy/branch/py12/py/_plugin/pytest_pdb.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_pdb.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,105 @@ +""" +interactive debugging with the Python Debugger. +""" +import py +import pdb, sys, linecache + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--pdb', + action="store_true", dest="usepdb", default=False, + help="start the interactive Python debugger on errors.") + +def pytest_configure(config): + if config.getvalue("usepdb"): + config.pluginmanager.register(PdbInvoke(), 'pdb') + +class PdbInvoke: + def pytest_runtest_makereport(self, item, call): + if call.excinfo and not \ + call.excinfo.errisinstance(py.test.skip.Exception): + # play well with capturing, slightly hackish + capman = item.config.pluginmanager.getplugin('capturemanager') + capman.suspendcapture() + + tw = py.io.TerminalWriter() + repr = call.excinfo.getrepr() + repr.toterminal(tw) + post_mortem(call.excinfo._excinfo[2]) + + capman.resumecapture_item(item) + +class Pdb(py.std.pdb.Pdb): + def do_list(self, arg): + self.lastcmd = 'list' + last = None + if arg: + try: + x = eval(arg, {}, {}) + if type(x) == type(()): + first, last = x + first = int(first) + last = int(last) + if last < first: + # Assume it's a count + last = first + last + else: + first = max(1, int(x) - 5) + except: + print ('*** Error in argument: %s' % repr(arg)) + return + elif self.lineno is None: + first = max(1, self.curframe.f_lineno - 5) + else: + first = self.lineno + 1 + if last is None: + last = first + 10 + filename = self.curframe.f_code.co_filename + breaklist = self.get_file_breaks(filename) + try: + for lineno in range(first, last+1): + # start difference from normal do_line + line = self._getline(filename, lineno) + # end difference from normal do_line + if not line: + print ('[EOF]') + break + else: + s = repr(lineno).rjust(3) + if len(s) < 4: s = s + ' ' + if lineno in breaklist: s = s + 'B' + else: s = s + ' ' + if lineno == self.curframe.f_lineno: + s = s + '->' + sys.stdout.write(s + '\t' + line) + self.lineno = lineno + except KeyboardInterrupt: + pass + do_l = do_list + + def _getline(self, filename, lineno): + if hasattr(filename, "__source__"): + try: + return filename.__source__.lines[lineno - 1] + "\n" + except IndexError: + return None + return linecache.getline(filename, lineno) + + def get_stack(self, f, t): + # Modified from bdb.py to be able to walk the stack beyond generators, + # which does not work in the normal pdb :-( + stack, i = pdb.Pdb.get_stack(self, f, t) + if f is None: + i = max(0, len(stack) - 1) + while i and stack[i][0].f_locals.get("__tracebackhide__", False): + i-=1 + return stack, i + +def post_mortem(t): + p = Pdb() + p.reset() + p.interaction(None, t) + +def set_trace(): + # again, a copy of the version in pdb.py + Pdb().set_trace(sys._getframe().f_back) Added: pypy/branch/py12/py/_plugin/pytest_pylint.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_pylint.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,36 @@ +"""pylint plugin + +XXX: Currently in progress, NOT IN WORKING STATE. +""" +import py + +pylint = py.test.importorskip("pylint.lint") + +def pytest_addoption(parser): + group = parser.getgroup('pylint options') + group.addoption('--pylint', action='store_true', + default=False, dest='pylint', + help='run pylint on python files.') + +def pytest_collect_file(path, parent): + if path.ext == ".py": + if parent.config.getvalue('pylint'): + return PylintItem(path, parent) + +#def pytest_terminal_summary(terminalreporter): +# print 'placeholder for pylint output' + +class PylintItem(py.test.collect.Item): + def runtest(self): + capture = py.io.StdCaptureFD() + try: + linter = pylint.lint.PyLinter() + linter.check(str(self.fspath)) + finally: + out, err = capture.reset() + rating = out.strip().split('\n')[-1] + sys.stdout.write(">>>") + print(rating) + assert 0 + + Added: pypy/branch/py12/py/_plugin/pytest_pytester.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_pytester.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,490 @@ +""" +funcargs and support code for testing py.test's own functionality. +""" + +import py +import sys, os +import re +import inspect +import time +from py._test.config import Config as pytestConfig +from py.builtin import print_ + +def pytest_addoption(parser): + group = parser.getgroup("pylib") + group.addoption('--tools-on-path', + action="store_true", dest="toolsonpath", default=False, + help=("discover tools on PATH instead of going through py.cmdline.") + ) + +pytest_plugins = '_pytest' + +def pytest_funcarg__linecomp(request): + return LineComp() + +def pytest_funcarg__LineMatcher(request): + return LineMatcher + +def pytest_funcarg__testdir(request): + tmptestdir = TmpTestdir(request) + return tmptestdir + +rex_outcome = re.compile("(\d+) (\w+)") +class RunResult: + def __init__(self, ret, outlines, errlines, duration): + self.ret = ret + self.outlines = outlines + self.errlines = errlines + self.stdout = LineMatcher(outlines) + self.stderr = LineMatcher(errlines) + self.duration = duration + + def parseoutcomes(self): + for line in reversed(self.outlines): + if 'seconds' in line: + outcomes = rex_outcome.findall(line) + if outcomes: + d = {} + for num, cat in outcomes: + d[cat] = int(num) + return d + +class TmpTestdir: + def __init__(self, request): + self.request = request + self._pytest = request.getfuncargvalue("_pytest") + # XXX remove duplication with tmpdir plugin + basetmp = request.config.ensuretemp("testdir") + name = request.function.__name__ + for i in range(100): + try: + tmpdir = basetmp.mkdir(name + str(i)) + except py.error.EEXIST: + continue + break + # we need to create another subdir + # because Directory.collect() currently loads + # conftest.py from sibling directories + self.tmpdir = tmpdir.mkdir(name) + self.plugins = [] + self._syspathremove = [] + self.chdir() # always chdir + self.request.addfinalizer(self.finalize) + + def __repr__(self): + return "" % (self.tmpdir,) + + def Config(self, topdir=None): + if topdir is None: + topdir = self.tmpdir.dirpath() + return pytestConfig(topdir=topdir) + + def finalize(self): + for p in self._syspathremove: + py.std.sys.path.remove(p) + if hasattr(self, '_olddir'): + self._olddir.chdir() + # delete modules that have been loaded from tmpdir + for name, mod in list(sys.modules.items()): + if mod: + fn = getattr(mod, '__file__', None) + if fn and fn.startswith(str(self.tmpdir)): + del sys.modules[name] + + def getreportrecorder(self, obj): + if hasattr(obj, 'config'): + obj = obj.config + if hasattr(obj, 'hook'): + obj = obj.hook + assert hasattr(obj, '_hookspecs'), obj + reprec = ReportRecorder(obj) + reprec.hookrecorder = self._pytest.gethookrecorder(obj) + reprec.hook = reprec.hookrecorder.hook + return reprec + + def chdir(self): + old = self.tmpdir.chdir() + if not hasattr(self, '_olddir'): + self._olddir = old + + def _makefile(self, ext, args, kwargs): + items = list(kwargs.items()) + if args: + source = "\n".join(map(str, args)) + basename = self.request.function.__name__ + items.insert(0, (basename, source)) + ret = None + for name, value in items: + p = self.tmpdir.join(name).new(ext=ext) + source = str(py.code.Source(value)).lstrip() + p.write(source.encode("utf-8"), "wb") + if ret is None: + ret = p + return ret + + + def makefile(self, ext, *args, **kwargs): + return self._makefile(ext, args, kwargs) + + def makeconftest(self, source): + return self.makepyfile(conftest=source) + + def makepyfile(self, *args, **kwargs): + return self._makefile('.py', args, kwargs) + + def maketxtfile(self, *args, **kwargs): + return self._makefile('.txt', args, kwargs) + + def syspathinsert(self, path=None): + if path is None: + path = self.tmpdir + py.std.sys.path.insert(0, str(path)) + self._syspathremove.append(str(path)) + + def mkdir(self, name): + return self.tmpdir.mkdir(name) + + def mkpydir(self, name): + p = self.mkdir(name) + p.ensure("__init__.py") + return p + + def genitems(self, colitems): + return list(self.session.genitems(colitems)) + + def inline_genitems(self, *args): + #config = self.parseconfig(*args) + config = self.parseconfig(*args) + session = config.initsession() + rec = self.getreportrecorder(config) + colitems = [config.getnode(arg) for arg in config.args] + items = list(session.genitems(colitems)) + return items, rec + + def runitem(self, source): + # used from runner functional tests + item = self.getitem(source) + # the test class where we are called from wants to provide the runner + testclassinstance = py.builtin._getimself(self.request.function) + runner = testclassinstance.getrunner() + return runner(item) + + def inline_runsource(self, source, *cmdlineargs): + p = self.makepyfile(source) + l = list(cmdlineargs) + [p] + return self.inline_run(*l) + + def inline_runsource1(self, *args): + args = list(args) + source = args.pop() + p = self.makepyfile(source) + l = list(args) + [p] + reprec = self.inline_run(*l) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 1, reports + return reports[0] + + def inline_run(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + session = config.initsession() + reprec = self.getreportrecorder(config) + colitems = config.getinitialnodes() + session.main(colitems) + config.pluginmanager.do_unconfigure(config) + return reprec + + def config_preparse(self): + config = self.Config() + for plugin in self.plugins: + if isinstance(plugin, str): + config.pluginmanager.import_plugin(plugin) + else: + if isinstance(plugin, dict): + plugin = PseudoPlugin(plugin) + if not config.pluginmanager.isregistered(plugin): + config.pluginmanager.register(plugin) + return config + + def parseconfig(self, *args): + if not args: + args = (self.tmpdir,) + config = self.config_preparse() + args = list(args) + ["--basetemp=%s" % self.tmpdir.dirpath('basetemp')] + config.parse(args) + return config + + def reparseconfig(self, args=None): + """ this is used from tests that want to re-invoke parse(). """ + if not args: + args = [self.tmpdir] + from py._test import config + oldconfig = config.config_per_process # py.test.config + try: + c = config.config_per_process = py.test.config = pytestConfig() + c.basetemp = oldconfig.mktemp("reparse", numbered=True) + c.parse(args) + return c + finally: + config.config_per_process = py.test.config = oldconfig + + def parseconfigure(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + return config + + def getitem(self, source, funcname="test_func"): + modcol = self.getmodulecol(source) + moditems = modcol.collect() + for item in modcol.collect(): + if item.name == funcname: + return item + else: + assert 0, "%r item not found in module:\n%s" %(funcname, source) + + def getitems(self, source): + modcol = self.getmodulecol(source) + return list(modcol.config.initsession().genitems([modcol])) + #assert item is not None, "%r item not found in module:\n%s" %(funcname, source) + #return item + + def getfscol(self, path, configargs=()): + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + return self.config.getnode(path) + + def getmodulecol(self, source, configargs=(), withinit=False): + kw = {self.request.function.__name__: py.code.Source(source).strip()} + path = self.makepyfile(**kw) + if withinit: + self.makepyfile(__init__ = "#") + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + #self.config.pluginmanager.do_configure(config=self.config) + # XXX + self.config.pluginmanager.import_plugin("runner") + plugin = self.config.pluginmanager.getplugin("runner") + plugin.pytest_configure(config=self.config) + + return self.config.getnode(path) + + def popen(self, cmdargs, stdout, stderr, **kw): + if not hasattr(py.std, 'subprocess'): + py.test.skip("no subprocess module") + env = os.environ.copy() + env['PYTHONPATH'] = ":".join(filter(None, [ + str(os.getcwd()), env.get('PYTHONPATH', '')])) + kw['env'] = env + #print "env", env + return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw) + + def run(self, *cmdargs): + return self._run(*cmdargs) + + def _run(self, *cmdargs): + cmdargs = [str(x) for x in cmdargs] + p1 = self.tmpdir.join("stdout") + p2 = self.tmpdir.join("stderr") + print_("running", cmdargs, "curdir=", py.path.local()) + f1 = p1.open("wb") + f2 = p2.open("wb") + now = time.time() + popen = self.popen(cmdargs, stdout=f1, stderr=f2, + close_fds=(sys.platform != "win32")) + ret = popen.wait() + f1.close() + f2.close() + out = p1.read("rb").decode("utf-8").splitlines() + err = p2.read("rb").decode("utf-8").splitlines() + def dump_lines(lines, fp): + try: + for line in lines: + py.builtin.print_(line, file=fp) + except UnicodeEncodeError: + print("couldn't print to %s because of encoding" % (fp,)) + dump_lines(out, sys.stdout) + dump_lines(err, sys.stderr) + return RunResult(ret, out, err, time.time()-now) + + def runpybin(self, scriptname, *args): + fullargs = self._getpybinargs(scriptname) + args + return self.run(*fullargs) + + def _getpybinargs(self, scriptname): + if self.request.config.getvalue("toolsonpath"): + script = py.path.local.sysfind(scriptname) + assert script, "script %r not found" % scriptname + return (script,) + else: + cmdlinename = scriptname.replace(".", "") + assert hasattr(py.cmdline, cmdlinename), cmdlinename + source = ("import sys;sys.path.insert(0,%r);" + "import py;py.cmdline.%s()" % + (str(py._pydir.dirpath()), cmdlinename)) + return (sys.executable, "-c", source,) + + def runpython(self, script): + s = self._getsysprepend() + if s: + script.write(s + "\n" + script.read()) + return self.run(sys.executable, script) + + def _getsysprepend(self): + if not self.request.config.getvalue("toolsonpath"): + s = "import sys;sys.path.insert(0,%r);" % str(py._pydir.dirpath()) + else: + s = "" + return s + + def runpython_c(self, command): + command = self._getsysprepend() + command + return self.run(py.std.sys.executable, "-c", command) + + def runpytest(self, *args): + p = py.path.local.make_numbered_dir(prefix="runpytest-", + keep=None, rootdir=self.tmpdir) + args = ('--basetemp=%s' % p, ) + args + plugins = [x for x in self.plugins if isinstance(x, str)] + if plugins: + args = ('-p', plugins[0]) + args + return self.runpybin("py.test", *args) + + def spawn_pytest(self, string, expect_timeout=10.0): + pexpect = py.test.importorskip("pexpect", "2.4") + if not self.request.config.getvalue("toolsonpath"): + py.test.skip("need --tools-on-path to run py.test script") + basetemp = self.tmpdir.mkdir("pexpect") + invoke = self._getpybinargs("py.test")[0] + cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string) + child = pexpect.spawn(cmd, logfile=basetemp.join("spawn.out").open("w")) + child.timeout = expect_timeout + return child + +class PseudoPlugin: + def __init__(self, vars): + self.__dict__.update(vars) + +class ReportRecorder(object): + def __init__(self, hook): + self.hook = hook + self.registry = hook._registry + self.registry.register(self) + + def getcall(self, name): + return self.hookrecorder.getcall(name) + + def popcall(self, name): + return self.hookrecorder.popcall(name) + + def getcalls(self, names): + """ return list of ParsedCall instances matching the given eventname. """ + return self.hookrecorder.getcalls(names) + + # functionality for test reports + + def getreports(self, names="pytest_runtest_logreport pytest_collectreport"): + return [x.report for x in self.getcalls(names)] + + def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport"): + """ return a testreport whose dotted import path matches """ + l = [] + for rep in self.getreports(names=names): + colitem = rep.getnode() + if not inamepart or inamepart in colitem.listnames(): + l.append(rep) + if not l: + raise ValueError("could not find test report matching %r: no test reports at all!" % + (inamepart,)) + if len(l) > 1: + raise ValueError("found more than one testreport matching %r: %s" %( + inamepart, l)) + return l[0] + + def getfailures(self, names='pytest_runtest_logreport pytest_collectreport'): + return [rep for rep in self.getreports(names) if rep.failed] + + def getfailedcollections(self): + return self.getfailures('pytest_collectreport') + + def listoutcomes(self): + passed = [] + skipped = [] + failed = [] + for rep in self.getreports("pytest_runtest_logreport"): + if rep.passed: + if rep.when == "call": + passed.append(rep) + elif rep.skipped: + skipped.append(rep) + elif rep.failed: + failed.append(rep) + return passed, skipped, failed + + def countoutcomes(self): + return [len(x) for x in self.listoutcomes()] + + def assertoutcome(self, passed=0, skipped=0, failed=0): + realpassed, realskipped, realfailed = self.listoutcomes() + assert passed == len(realpassed) + assert skipped == len(realskipped) + assert failed == len(realfailed) + + def clear(self): + self.hookrecorder.calls[:] = [] + + def unregister(self): + self.registry.unregister(self) + self.hookrecorder.finish_recording() + +class LineComp: + def __init__(self): + self.stringio = py.io.TextIO() + + def assert_contains_lines(self, lines2): + """ assert that lines2 are contained (linearly) in lines1. + return a list of extralines found. + """ + __tracebackhide__ = True + val = self.stringio.getvalue() + self.stringio.truncate(0) + self.stringio.seek(0) + lines1 = val.split("\n") + return LineMatcher(lines1).fnmatch_lines(lines2) + +class LineMatcher: + def __init__(self, lines): + self.lines = lines + + def str(self): + return "\n".join(self.lines) + + def fnmatch_lines(self, lines2): + if isinstance(lines2, str): + lines2 = py.code.Source(lines2) + if isinstance(lines2, py.code.Source): + lines2 = lines2.strip().lines + + from fnmatch import fnmatch + lines1 = self.lines[:] + nextline = None + extralines = [] + __tracebackhide__ = True + for line in lines2: + nomatchprinted = False + while lines1: + nextline = lines1.pop(0) + if line == nextline: + print_("exact match:", repr(line)) + break + elif fnmatch(nextline, line): + print_("fnmatch:", repr(line)) + print_(" with:", repr(nextline)) + break + else: + if not nomatchprinted: + print_("nomatch:", repr(line)) + nomatchprinted = True + print_(" and:", repr(nextline)) + extralines.append(nextline) + else: + assert line == nextline Added: pypy/branch/py12/py/_plugin/pytest_pytester.py.orig ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_pytester.py.orig Fri Apr 30 17:07:52 2010 @@ -0,0 +1,494 @@ +""" +funcargs and support code for testing py.test's own functionality. +""" + +import py +import sys, os +import re +import inspect +import time +from py._test.config import Config as pytestConfig +from py.builtin import print_ + +def pytest_addoption(parser): + group = parser.getgroup("pylib") + group.addoption('--tools-on-path', + action="store_true", dest="toolsonpath", default=False, + help=("discover tools on PATH instead of going through py.cmdline.") + ) + +pytest_plugins = '_pytest' + +def pytest_funcarg__linecomp(request): + return LineComp() + +def pytest_funcarg__LineMatcher(request): + return LineMatcher + +def pytest_funcarg__testdir(request): + tmptestdir = TmpTestdir(request) + return tmptestdir + +rex_outcome = re.compile("(\d+) (\w+)") +class RunResult: + def __init__(self, ret, outlines, errlines, duration): + self.ret = ret + self.outlines = outlines + self.errlines = errlines + self.stdout = LineMatcher(outlines) + self.stderr = LineMatcher(errlines) + self.duration = duration + + def parseoutcomes(self): + for line in reversed(self.outlines): + if 'seconds' in line: + outcomes = rex_outcome.findall(line) + if outcomes: + d = {} + for num, cat in outcomes: + d[cat] = int(num) + return d + +class TmpTestdir: + def __init__(self, request): + self.request = request + self._pytest = request.getfuncargvalue("_pytest") + # XXX remove duplication with tmpdir plugin + basetmp = request.config.ensuretemp("testdir") + name = request.function.__name__ + for i in range(100): + try: + tmpdir = basetmp.mkdir(name + str(i)) + except py.error.EEXIST: + continue + break + # we need to create another subdir + # because Directory.collect() currently loads + # conftest.py from sibling directories + self.tmpdir = tmpdir.mkdir(name) + self.plugins = [] + self._syspathremove = [] + self.chdir() # always chdir + self.request.addfinalizer(self.finalize) + + def __repr__(self): + return "" % (self.tmpdir,) + + def Config(self, topdir=None): + if topdir is None: + topdir = self.tmpdir.dirpath() + return pytestConfig(topdir=topdir) + + def finalize(self): + for p in self._syspathremove: + py.std.sys.path.remove(p) + if hasattr(self, '_olddir'): + self._olddir.chdir() + # delete modules that have been loaded from tmpdir + for name, mod in list(sys.modules.items()): + if mod: + fn = getattr(mod, '__file__', None) + if fn and fn.startswith(str(self.tmpdir)): + del sys.modules[name] + + def getreportrecorder(self, obj): + if hasattr(obj, 'config'): + obj = obj.config + if hasattr(obj, 'hook'): + obj = obj.hook + assert hasattr(obj, '_hookspecs'), obj + reprec = ReportRecorder(obj) + reprec.hookrecorder = self._pytest.gethookrecorder(obj) + reprec.hook = reprec.hookrecorder.hook + return reprec + + def chdir(self): + old = self.tmpdir.chdir() + if not hasattr(self, '_olddir'): + self._olddir = old + + def _makefile(self, ext, args, kwargs): + items = list(kwargs.items()) + if args: + source = "\n".join(map(str, args)) + basename = self.request.function.__name__ + items.insert(0, (basename, source)) + ret = None + for name, value in items: + p = self.tmpdir.join(name).new(ext=ext) + source = py.code.Source(value) + p.write(str(py.code.Source(value)).lstrip()) + if ret is None: + ret = p + return ret + + + def makefile(self, ext, *args, **kwargs): + return self._makefile(ext, args, kwargs) + + def makeconftest(self, source): + return self.makepyfile(conftest=source) + + def makepyfile(self, *args, **kwargs): + return self._makefile('.py', args, kwargs) + + def maketxtfile(self, *args, **kwargs): + return self._makefile('.txt', args, kwargs) + + def syspathinsert(self, path=None): + if path is None: + path = self.tmpdir + py.std.sys.path.insert(0, str(path)) + self._syspathremove.append(str(path)) + + def mkdir(self, name): + return self.tmpdir.mkdir(name) + + def mkpydir(self, name): + p = self.mkdir(name) + p.ensure("__init__.py") + return p + + def genitems(self, colitems): + return list(self.session.genitems(colitems)) + + def inline_genitems(self, *args): + #config = self.parseconfig(*args) + config = self.parseconfig(*args) + session = config.initsession() + rec = self.getreportrecorder(config) + colitems = [config.getnode(arg) for arg in config.args] + items = list(session.genitems(colitems)) + return items, rec + + def runitem(self, source): + # used from runner functional tests + item = self.getitem(source) + # the test class where we are called from wants to provide the runner + testclassinstance = py.builtin._getimself(self.request.function) + runner = testclassinstance.getrunner() + return runner(item) + + def inline_runsource(self, source, *cmdlineargs): + p = self.makepyfile(source) + l = list(cmdlineargs) + [p] + return self.inline_run(*l) + + def inline_runsource1(self, *args): + args = list(args) + source = args.pop() + p = self.makepyfile(source) + l = list(args) + [p] + reprec = self.inline_run(*l) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 1, reports + return reports[0] + + def inline_run(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + session = config.initsession() + reprec = self.getreportrecorder(config) + colitems = config.getinitialnodes() + session.main(colitems) + config.pluginmanager.do_unconfigure(config) + return reprec + + def config_preparse(self): + config = self.Config() + for plugin in self.plugins: + if isinstance(plugin, str): + config.pluginmanager.import_plugin(plugin) + else: + if isinstance(plugin, dict): + plugin = PseudoPlugin(plugin) + if not config.pluginmanager.isregistered(plugin): + config.pluginmanager.register(plugin) + return config + + def parseconfig(self, *args): + if not args: + args = (self.tmpdir,) + config = self.config_preparse() + args = list(args) + ["--basetemp=%s" % self.tmpdir.dirpath('basetemp')] + config.parse(args) + return config + + def reparseconfig(self, args=None): + """ this is used from tests that want to re-invoke parse(). """ + if not args: + args = [self.tmpdir] + from py._test import config + oldconfig = config.config_per_process # py.test.config + try: + c = config.config_per_process = py.test.config = pytestConfig() + c.basetemp = oldconfig.mktemp("reparse", numbered=True) + c.parse(args) + return c + finally: + config.config_per_process = py.test.config = oldconfig + + def parseconfigure(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + return config + + def getitem(self, source, funcname="test_func"): + modcol = self.getmodulecol(source) + moditems = modcol.collect() + for item in modcol.collect(): + if item.name == funcname: + return item + else: + assert 0, "%r item not found in module:\n%s" %(funcname, source) + + def getitems(self, source): + modcol = self.getmodulecol(source) + return list(modcol.config.initsession().genitems([modcol])) + #assert item is not None, "%r item not found in module:\n%s" %(funcname, source) + #return item + + def getfscol(self, path, configargs=()): + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + return self.config.getnode(path) + + def getmodulecol(self, source, configargs=(), withinit=False): + kw = {self.request.function.__name__: py.code.Source(source).strip()} + path = self.makepyfile(**kw) + if withinit: + self.makepyfile(__init__ = "#") + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + #self.config.pluginmanager.do_configure(config=self.config) + # XXX + self.config.pluginmanager.import_plugin("runner") + plugin = self.config.pluginmanager.getplugin("runner") + plugin.pytest_configure(config=self.config) + + return self.config.getnode(path) + + def popen(self, cmdargs, stdout, stderr, **kw): + if not hasattr(py.std, 'subprocess'): + py.test.skip("no subprocess module") + env = os.environ.copy() + env['PYTHONPATH'] = ":".join(filter(None, [ + str(os.getcwd()), env.get('PYTHONPATH', '')])) + kw['env'] = env + #print "env", env + return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw) + + def run(self, *cmdargs): + return self._run(*cmdargs) + + def _run(self, *cmdargs): + cmdargs = [str(x) for x in cmdargs] + p1 = self.tmpdir.join("stdout") + p2 = self.tmpdir.join("stderr") + print_("running", cmdargs, "curdir=", py.path.local()) + # we don't write to a file because Jython2.5.1 will internally + # anyway open PIPEs which subsequently leak leading to + # Too-Many-Open-Files + PIPE = py.std.subprocess.PIPE + now = time.time() + popen = self.popen(cmdargs, stdout=PIPE, stderr=PIPE, ) + #close_fds=(sys.platform != "win32")) + p1.write(popen.stdout.read()) + p2.write(popen.stderr.read()) + popen.stdout.close() + popen.stderr.close() + ret = popen.wait() + out, err = p1.readlines(cr=0), p2.readlines(cr=0) + #if err: + # for line in err: + # py.builtin.print_(line, file=sys.stderr) + #if out: + # for line in out: + # py.builtin.print_(line, file=sys.stdout) + return RunResult(ret, out, err, time.time()-now) + + def runpybin(self, scriptname, *args): + fullargs = self._getpybinargs(scriptname) + args + return self.run(*fullargs) + + def _getpybinargs(self, scriptname): + if self.request.config.getvalue("toolsonpath"): + script = py.path.local.sysfind(scriptname) + assert script, "script %r not found" % scriptname + return (script,) + else: + cmdlinename = scriptname.replace(".", "") + assert hasattr(py.cmdline, cmdlinename), cmdlinename + source = ("import sys;sys.path.insert(0,%r);" + "import py;py.cmdline.%s()" % + (str(py._pydir.dirpath()), cmdlinename)) + return (sys.executable, "-c", source,) + + def runpython(self, script): + s = self._getsysprepend() + if s: + script.write(s + "\n" + script.read()) + return self.run(sys.executable, script) + + def _getsysprepend(self): + if not self.request.config.getvalue("toolsonpath"): + s = "import sys;sys.path.insert(0,%r);" % str(py._pydir.dirpath()) + else: + s = "" + return s + + def runpython_c(self, command): + command = self._getsysprepend() + command + return self.run(py.std.sys.executable, "-c", command) + + def runpytest(self, *args): + p = py.path.local.make_numbered_dir(prefix="runpytest-", + keep=None, rootdir=self.tmpdir) + args = ('--basetemp=%s' % p, ) + args + plugins = [x for x in self.plugins if isinstance(x, str)] + if plugins: + args = ('-p', plugins[0]) + args + return self.runpybin("py.test", *args) + + def spawn_pytest(self, string, expect_timeout=10.0): + pexpect = py.test.importorskip("pexpect", "2.4") + if not self.request.config.getvalue("toolsonpath"): + py.test.skip("need --tools-on-path to run py.test script") + basetemp = self.tmpdir.mkdir("pexpect") + invoke = self._getpybinargs("py.test")[0] + cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string) + child = pexpect.spawn(cmd, logfile=basetemp.join("spawn.out").open("w")) + child.timeout = expect_timeout + return child + +class PseudoPlugin: + def __init__(self, vars): + self.__dict__.update(vars) + +class ReportRecorder(object): + def __init__(self, hook): + self.hook = hook + self.registry = hook._registry + self.registry.register(self) + + def getcall(self, name): + return self.hookrecorder.getcall(name) + + def popcall(self, name): + return self.hookrecorder.popcall(name) + + def getcalls(self, names): + """ return list of ParsedCall instances matching the given eventname. """ + return self.hookrecorder.getcalls(names) + + # functionality for test reports + + def getreports(self, names="pytest_runtest_logreport pytest_collectreport"): + return [x.report for x in self.getcalls(names)] + + def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport"): + """ return a testreport whose dotted import path matches """ + l = [] + for rep in self.getreports(names=names): + colitem = rep.getnode() + if not inamepart or inamepart in colitem.listnames(): + l.append(rep) + if not l: + raise ValueError("could not find test report matching %r: no test reports at all!" % + (inamepart,)) + if len(l) > 1: + raise ValueError("found more than one testreport matching %r: %s" %( + inamepart, l)) + return l[0] + + def getfailures(self, names='pytest_runtest_logreport pytest_collectreport'): + return [rep for rep in self.getreports(names) if rep.failed] + + def getfailedcollections(self): + return self.getfailures('pytest_collectreport') + + def listoutcomes(self): + passed = [] + skipped = [] + failed = [] + for rep in self.getreports("pytest_runtest_logreport"): + if rep.passed: + if rep.when == "call": + passed.append(rep) + elif rep.skipped: + skipped.append(rep) + elif rep.failed: + failed.append(rep) + return passed, skipped, failed + + def countoutcomes(self): + return [len(x) for x in self.listoutcomes()] + + def assertoutcome(self, passed=0, skipped=0, failed=0): + realpassed, realskipped, realfailed = self.listoutcomes() + assert passed == len(realpassed) + assert skipped == len(realskipped) + assert failed == len(realfailed) + + def clear(self): + self.hookrecorder.calls[:] = [] + + def unregister(self): + self.registry.unregister(self) + self.hookrecorder.finish_recording() + +class LineComp: + def __init__(self): + self.stringio = py.io.TextIO() + + def assert_contains_lines(self, lines2): + """ assert that lines2 are contained (linearly) in lines1. + return a list of extralines found. + """ + __tracebackhide__ = True + val = self.stringio.getvalue() + self.stringio.truncate(0) # remove what we got + lines1 = val.split("\n") + return LineMatcher(lines1).fnmatch_lines(lines2) + +class LineMatcher: + def __init__(self, lines): + self.lines = lines + + def str(self): + return "\n".join(self.lines) + + def fnmatch_lines(self, lines2): + if isinstance(lines2, str): + lines2 = py.code.Source(lines2) + if isinstance(lines2, py.code.Source): + lines2 = lines2.strip().lines + + from fnmatch import fnmatch + __tracebackhide__ = True + lines1 = self.lines[:] + nextline = None + extralines = [] + for line in lines2: + nomatchprinted = False + while lines1: + nextline = lines1.pop(0) + if line == nextline: + print_("exact match:", repr(line)) + break + elif fnmatch(nextline, line): + print_("fnmatch:", repr(line)) + print_(" with:", repr(nextline)) + break + else: + if not nomatchprinted: + print_("nomatch:", repr(line)) + nomatchprinted = True + print_(" and:", repr(nextline)) + extralines.append(nextline) + else: + if line != nextline: + #__tracebackhide__ = True + raise AssertionError("expected line not found: %r" % line) + extralines.extend(lines1) + return extralines Added: pypy/branch/py12/py/_plugin/pytest_recwarn.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_recwarn.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,121 @@ +""" +helpers for asserting deprecation and other warnings. + +Example usage +--------------------- + +You can use the ``recwarn`` funcarg to track +warnings within a test function: + +.. sourcecode:: python + + def test_hello(recwarn): + from warnings import warn + warn("hello", DeprecationWarning) + w = recwarn.pop(DeprecationWarning) + assert issubclass(w.category, DeprecationWarning) + assert 'hello' in str(w.message) + assert w.filename + assert w.lineno + +You can also call a global helper for checking +taht a certain function call yields a Deprecation +warning: + +.. sourcecode:: python + + import py + + def test_global(): + py.test.deprecated_call(myfunction, 17) + + +""" + +import py +import os + +def pytest_funcarg__recwarn(request): + """Return a WarningsRecorder instance that provides these methods: + + * ``pop(category=None)``: return last warning matching the category. + * ``clear()``: clear list of warnings + """ + warnings = WarningsRecorder() + request.addfinalizer(warnings.finalize) + return warnings + +def pytest_namespace(): + return {'deprecated_call': deprecated_call} + +def deprecated_call(func, *args, **kwargs): + """ assert that calling func(*args, **kwargs) + triggers a DeprecationWarning. + """ + warningmodule = py.std.warnings + l = [] + oldwarn_explicit = getattr(warningmodule, 'warn_explicit') + def warn_explicit(*args, **kwargs): + l.append(args) + oldwarn_explicit(*args, **kwargs) + oldwarn = getattr(warningmodule, 'warn') + def warn(*args, **kwargs): + l.append(args) + oldwarn(*args, **kwargs) + + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + try: + ret = func(*args, **kwargs) + finally: + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + if not l: + #print warningmodule + __tracebackhide__ = True + raise AssertionError("%r did not produce DeprecationWarning" %(func,)) + return ret + + +class RecordedWarning: + def __init__(self, message, category, filename, lineno, line): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.line = line + +class WarningsRecorder: + def __init__(self): + warningmodule = py.std.warnings + self.list = [] + def showwarning(message, category, filename, lineno, line=0): + self.list.append(RecordedWarning( + message, category, filename, lineno, line)) + try: + self.old_showwarning(message, category, + filename, lineno, line=line) + except TypeError: + # < python2.6 + self.old_showwarning(message, category, filename, lineno) + self.old_showwarning = warningmodule.showwarning + warningmodule.showwarning = showwarning + + def pop(self, cls=Warning): + """ pop the first recorded warning, raise exception if not exists.""" + for i, w in enumerate(self.list): + if issubclass(w.category, cls): + return self.list.pop(i) + __tracebackhide__ = True + assert 0, "%r not found in %r" %(cls, self.list) + + #def resetregistry(self): + # import warnings + # warnings.onceregistry.clear() + # warnings.__warningregistry__.clear() + + def clear(self): + self.list[:] = [] + + def finalize(self): + py.std.warnings.showwarning = self.old_showwarning Added: pypy/branch/py12/py/_plugin/pytest_restdoc.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_restdoc.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,429 @@ +""" +perform ReST syntax, local and remote reference tests on .rst/.txt files. +""" +import py +import sys, os, re + +def pytest_addoption(parser): + group = parser.getgroup("ReST", "ReST documentation check options") + group.addoption('-R', '--urlcheck', + action="store_true", dest="urlcheck", default=False, + help="urlopen() remote links found in ReST text files.") + group.addoption('--urltimeout', action="store", metavar="secs", + type="int", dest="urlcheck_timeout", default=5, + help="timeout in seconds for remote urlchecks") + group.addoption('--forcegen', + action="store_true", dest="forcegen", default=False, + help="force generation of html files.") + +def pytest_collect_file(path, parent): + if path.ext in (".txt", ".rst"): + project = getproject(path) + if project is not None: + return ReSTFile(path, parent=parent, project=project) + +def getproject(path): + for parent in path.parts(reverse=True): + confrest = parent.join("confrest.py") + if confrest.check(): + Project = confrest.pyimport().Project + return Project(parent) + +class ReSTFile(py.test.collect.File): + def __init__(self, fspath, parent, project): + super(ReSTFile, self).__init__(fspath=fspath, parent=parent) + self.project = project + + def collect(self): + return [ + ReSTSyntaxTest("ReSTSyntax", parent=self, project=self.project), + LinkCheckerMaker("checklinks", parent=self), + DoctestText("doctest", parent=self), + ] + +def deindent(s, sep='\n'): + leastspaces = -1 + lines = s.split(sep) + for line in lines: + if not line.strip(): + continue + spaces = len(line) - len(line.lstrip()) + if leastspaces == -1 or spaces < leastspaces: + leastspaces = spaces + if leastspaces == -1: + return s + for i, line in enumerate(lines): + if not line.strip(): + lines[i] = '' + else: + lines[i] = line[leastspaces:] + return sep.join(lines) + +class ReSTSyntaxTest(py.test.collect.Item): + def __init__(self, name, parent, project): + super(ReSTSyntaxTest, self).__init__(name=name, parent=parent) + self.project = project + + def reportinfo(self): + return self.fspath, None, "syntax check" + + def runtest(self): + self.restcheck(py.path.svnwc(self.fspath)) + + def restcheck(self, path): + py.test.importorskip("docutils") + self.register_linkrole() + from docutils.utils import SystemMessage + try: + self._checkskip(path, self.project.get_htmloutputpath(path)) + self.project.process(path) + except KeyboardInterrupt: + raise + except SystemMessage: + # we assume docutils printed info on stdout + py.test.fail("docutils processing failed, see captured stderr") + + def register_linkrole(self): + #directive.register_linkrole('api', self.resolve_linkrole) + #directive.register_linkrole('source', self.resolve_linkrole) +# +# # XXX fake sphinx' "toctree" and refs +# directive.register_linkrole('ref', self.resolve_linkrole) + + from docutils.parsers.rst import directives + def toctree_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + toctree_directive.content = 1 + toctree_directive.options = {'maxdepth': int, 'glob': directives.flag, + 'hidden': directives.flag} + directives.register_directive('toctree', toctree_directive) + self.register_pygments() + + def register_pygments(self): + # taken from pygments-main/external/rst-directive.py + from docutils.parsers.rst import directives + try: + from pygments.formatters import HtmlFormatter + except ImportError: + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + pygments_directive.options = {} + else: + # The default formatter + DEFAULT = HtmlFormatter(noclasses=True) + # Add name -> formatter pairs for every variant you want to use + VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), + } + + from docutils import nodes + + from pygments import highlight + from pygments.lexers import get_lexer_by_name, TextLexer + + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + try: + lexer = get_lexer_by_name(arguments[0]) + except ValueError: + # no lexer found - use the text one instead of an exception + lexer = TextLexer() + # take an arbitrary option if more than one is given + formatter = options and VARIANTS[options.keys()[0]] or DEFAULT + parsed = highlight('\n'.join(content), lexer, formatter) + return [nodes.raw('', parsed, format='html')] + + pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS]) + + pygments_directive.arguments = (1, 0, 1) + pygments_directive.content = 1 + directives.register_directive('sourcecode', pygments_directive) + + def resolve_linkrole(self, name, text, check=True): + apigen_relpath = self.project.apigen_relpath + + if name == 'api': + if text == 'py': + return ('py', apigen_relpath + 'api/index.html') + else: + assert text.startswith('py.'), ( + 'api link "%s" does not point to the py package') % (text,) + dotted_name = text + if dotted_name.find('(') > -1: + dotted_name = dotted_name[:text.find('(')] + # remove pkg root + path = dotted_name.split('.')[1:] + dotted_name = '.'.join(path) + obj = py + if check: + for chunk in path: + try: + obj = getattr(obj, chunk) + except AttributeError: + raise AssertionError( + 'problem with linkrole :api:`%s`: can not resolve ' + 'dotted name %s' % (text, dotted_name,)) + return (text, apigen_relpath + 'api/%s.html' % (dotted_name,)) + elif name == 'source': + assert text.startswith('py/'), ('source link "%s" does not point ' + 'to the py package') % (text,) + relpath = '/'.join(text.split('/')[1:]) + if check: + pkgroot = py._pydir + abspath = pkgroot.join(relpath) + assert pkgroot.join(relpath).check(), ( + 'problem with linkrole :source:`%s`: ' + 'path %s does not exist' % (text, relpath)) + if relpath.endswith('/') or not relpath: + relpath += 'index.html' + else: + relpath += '.html' + return (text, apigen_relpath + 'source/%s' % (relpath,)) + elif name == 'ref': + return ("", "") + + def _checkskip(self, lpath, htmlpath=None): + if not self.config.getvalue("forcegen"): + lpath = py.path.local(lpath) + if htmlpath is not None: + htmlpath = py.path.local(htmlpath) + if lpath.ext == '.txt': + htmlpath = htmlpath or lpath.new(ext='.html') + if htmlpath.check(file=1) and htmlpath.mtime() >= lpath.mtime(): + py.test.skip("html file is up to date, use --forcegen to regenerate") + #return [] # no need to rebuild + +class DoctestText(py.test.collect.Item): + def reportinfo(self): + return self.fspath, None, "doctest" + + def runtest(self): + content = self._normalize_linesep() + newcontent = self.config.hook.pytest_doctest_prepare_content(content=content) + if newcontent is not None: + content = newcontent + s = content + l = [] + prefix = '.. >>> ' + mod = py.std.types.ModuleType(self.fspath.purebasename) + skipchunk = False + for line in deindent(s).split('\n'): + stripped = line.strip() + if skipchunk and line.startswith(skipchunk): + py.builtin.print_("skipping", line) + continue + skipchunk = False + if stripped.startswith(prefix): + try: + py.builtin.exec_(py.code.Source( + stripped[len(prefix):]).compile(), mod.__dict__) + except ValueError: + e = sys.exc_info()[1] + if e.args and e.args[0] == "skipchunk": + skipchunk = " " * (len(line) - len(line.lstrip())) + else: + raise + else: + l.append(line) + docstring = "\n".join(l) + mod.__doc__ = docstring + failed, tot = py.std.doctest.testmod(mod, verbose=1) + if failed: + py.test.fail("doctest %s: %s failed out of %s" %( + self.fspath, failed, tot)) + + def _normalize_linesep(self): + # XXX quite nasty... but it works (fixes win32 issues) + s = self.fspath.read() + linesep = '\n' + if '\r' in s: + if '\n' not in s: + linesep = '\r' + else: + linesep = '\r\n' + s = s.replace(linesep, '\n') + return s + +class LinkCheckerMaker(py.test.collect.Collector): + def collect(self): + return list(self.genlinkchecks()) + + def genlinkchecks(self): + path = self.fspath + # generating functions + args as single tests + timeout = self.config.getvalue("urlcheck_timeout") + for lineno, line in enumerate(path.readlines()): + line = line.strip() + if line.startswith('.. _'): + if line.startswith('.. _`'): + delim = '`:' + else: + delim = ':' + l = line.split(delim, 1) + if len(l) != 2: + continue + tryfn = l[1].strip() + name = "%s:%d" %(tryfn, lineno) + if tryfn.startswith('http:') or tryfn.startswith('https'): + if self.config.getvalue("urlcheck"): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno, timeout), checkfunc=urlcheck) + elif tryfn.startswith('webcal:'): + continue + else: + i = tryfn.find('#') + if i != -1: + checkfn = tryfn[:i] + else: + checkfn = tryfn + if checkfn.strip() and (1 or checkfn.endswith('.html')): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno), checkfunc=localrefcheck) + +class CheckLink(py.test.collect.Item): + def __init__(self, name, parent, args, checkfunc): + super(CheckLink, self).__init__(name, parent) + self.args = args + self.checkfunc = checkfunc + + def runtest(self): + return self.checkfunc(*self.args) + + def reportinfo(self, basedir=None): + return (self.fspath, self.args[2], "checklink: %s" % self.args[0]) + +def urlcheck(tryfn, path, lineno, TIMEOUT_URLOPEN): + old = py.std.socket.getdefaulttimeout() + py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN) + try: + try: + py.builtin.print_("trying remote", tryfn) + py.std.urllib2.urlopen(tryfn) + finally: + py.std.socket.setdefaulttimeout(old) + except (py.std.urllib2.URLError, py.std.urllib2.HTTPError): + e = sys.exc_info()[1] + if getattr(e, 'code', None) in (401, 403): # authorization required, forbidden + py.test.skip("%s: %s" %(tryfn, str(e))) + else: + py.test.fail("remote reference error %r in %s:%d\n%s" %( + tryfn, path.basename, lineno+1, e)) + +def localrefcheck(tryfn, path, lineno): + # assume it should be a file + i = tryfn.find('#') + if tryfn.startswith('javascript:'): + return # don't check JS refs + if i != -1: + anchor = tryfn[i+1:] + tryfn = tryfn[:i] + else: + anchor = '' + fn = path.dirpath(tryfn) + ishtml = fn.ext == '.html' + fn = ishtml and fn.new(ext='.txt') or fn + py.builtin.print_("filename is", fn) + if not fn.check(): # not ishtml or not fn.check(): + if not py.path.local(tryfn).check(): # the html could be there + py.test.fail("reference error %r in %s:%d" %( + tryfn, path.basename, lineno+1)) + if anchor: + source = unicode(fn.read(), 'latin1') + source = source.lower().replace('-', ' ') # aehem + + anchor = anchor.replace('-', ' ') + match2 = ".. _`%s`:" % anchor + match3 = ".. _%s:" % anchor + candidates = (anchor, match2, match3) + py.builtin.print_("candidates", repr(candidates)) + for line in source.split('\n'): + line = line.strip() + if line in candidates: + break + else: + py.test.fail("anchor reference error %s#%s in %s:%d" %( + tryfn, anchor, path.basename, lineno+1)) + +if hasattr(sys.stdout, 'fileno') and os.isatty(sys.stdout.fileno()): + def log(msg): + print(msg) +else: + def log(msg): + pass + +def convert_rest_html(source, source_path, stylesheet=None, encoding='latin1'): + """ return html latin1-encoded document for the given input. + source a ReST-string + sourcepath where to look for includes (basically) + stylesheet path (to be used if any) + """ + from docutils.core import publish_string + kwargs = { + 'stylesheet' : stylesheet, + 'stylesheet_path': None, + 'traceback' : 1, + 'embed_stylesheet': 0, + 'output_encoding' : encoding, + #'halt' : 0, # 'info', + 'halt_level' : 2, + } + # docutils uses os.getcwd() :-( + source_path = os.path.abspath(str(source_path)) + prevdir = os.getcwd() + try: + #os.chdir(os.path.dirname(source_path)) + return publish_string(source, source_path, writer_name='html', + settings_overrides=kwargs) + finally: + os.chdir(prevdir) + +def process(txtpath, encoding='latin1'): + """ process a textfile """ + log("processing %s" % txtpath) + assert txtpath.check(ext='.txt') + if isinstance(txtpath, py.path.svnwc): + txtpath = txtpath.localpath + htmlpath = txtpath.new(ext='.html') + #svninfopath = txtpath.localpath.new(ext='.svninfo') + + style = txtpath.dirpath('style.css') + if style.check(): + stylesheet = style.basename + else: + stylesheet = None + content = unicode(txtpath.read(), encoding) + doc = convert_rest_html(content, txtpath, stylesheet=stylesheet, encoding=encoding) + htmlpath.open('wb').write(doc) + #log("wrote %r" % htmlpath) + #if txtpath.check(svnwc=1, versioned=1): + # info = txtpath.info() + # svninfopath.dump(info) + +if sys.version_info > (3, 0): + def _uni(s): return s +else: + def _uni(s): + return unicode(s) + +rex1 = re.compile(r'.*(.*).*', re.MULTILINE | re.DOTALL) +rex2 = re.compile(r'.*
(.*)
.*', re.MULTILINE | re.DOTALL) + +def strip_html_header(string, encoding='utf8'): + """ return the content of the body-tag """ + uni = unicode(string, encoding) + for rex in rex1,rex2: + match = rex.search(uni) + if not match: + break + uni = match.group(1) + return uni + +class Project: # used for confrest.py files + def __init__(self, sourcepath): + self.sourcepath = sourcepath + def process(self, path): + return process(path) + def get_htmloutputpath(self, path): + return path.new(ext='html') Added: pypy/branch/py12/py/_plugin/pytest_resultlog.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_resultlog.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,98 @@ +"""non-xml machine-readable logging of test results. + Useful for buildbot integration code. See the `PyPy-test`_ + web page for post-processing. + +.. _`PyPy-test`: http://codespeak.net:8099/summary + +""" + +import py +from py.builtin import print_ + +def pytest_addoption(parser): + group = parser.getgroup("resultlog", "resultlog plugin options") + group.addoption('--resultlog', action="store", dest="resultlog", metavar="path", default=None, + help="path for machine-readable result log.") + +def pytest_configure(config): + resultlog = config.option.resultlog + if resultlog: + logfile = open(resultlog, 'w', 1) # line buffered + config._resultlog = ResultLog(config, logfile) + config.pluginmanager.register(config._resultlog) + +def pytest_unconfigure(config): + resultlog = getattr(config, '_resultlog', None) + if resultlog: + resultlog.logfile.close() + del config._resultlog + config.pluginmanager.unregister(resultlog) + +def generic_path(item): + chain = item.listchain() + gpath = [chain[0].name] + fspath = chain[0].fspath + fspart = False + for node in chain[1:]: + newfspath = node.fspath + if newfspath == fspath: + if fspart: + gpath.append(':') + fspart = False + else: + gpath.append('.') + else: + gpath.append('/') + fspart = True + name = node.name + if name[0] in '([': + gpath.pop() + gpath.append(name) + fspath = newfspath + return ''.join(gpath) + +class ResultLog(object): + def __init__(self, config, logfile): + self.config = config + self.logfile = logfile # preferably line buffered + + def write_log_entry(self, testpath, shortrepr, longrepr): + print_("%s %s" % (shortrepr, testpath), file=self.logfile) + for line in longrepr.splitlines(): + print_(" %s" % line, file=self.logfile) + + def log_outcome(self, node, shortrepr, longrepr): + testpath = generic_path(node) + self.write_log_entry(testpath, shortrepr, longrepr) + + def pytest_runtest_logreport(self, report): + res = self.config.hook.pytest_report_teststatus(report=report) + if res is not None: + code = res[1] + else: + code = report.shortrepr + if code == 'x': + longrepr = str(report.longrepr) + elif code == 'P': + longrepr = '' + elif report.passed: + longrepr = "" + elif report.failed: + longrepr = str(report.longrepr) + elif report.skipped: + longrepr = str(report.longrepr.reprcrash.message) + self.log_outcome(report.item, code, longrepr) + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + code = "F" + else: + assert report.skipped + code = "S" + longrepr = str(report.longrepr.reprcrash) + self.log_outcome(report.collector, code, longrepr) + + def pytest_internalerror(self, excrepr): + path = excrepr.reprcrash.path + self.write_log_entry(path, '!', str(excrepr)) Added: pypy/branch/py12/py/_plugin/pytest_runner.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_runner.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,400 @@ +""" +collect and run test items and create reports. +""" + +import py, sys + +def pytest_namespace(): + return { + 'raises' : raises, + 'skip' : skip, + 'importorskip' : importorskip, + 'fail' : fail, + 'exit' : exit, + } + +# +# pytest plugin hooks + +# XXX move to pytest_sessionstart and fix py.test owns tests +def pytest_configure(config): + config._setupstate = SetupState() + +def pytest_sessionfinish(session, exitstatus): + if hasattr(session.config, '_setupstate'): + hook = session.config.hook + rep = hook.pytest__teardown_final(session=session) + if rep: + hook.pytest__teardown_final_logerror(report=rep) + +def pytest_make_collect_report(collector): + result = excinfo = None + try: + result = collector._memocollect() + except KeyboardInterrupt: + raise + except: + excinfo = py.code.ExceptionInfo() + return CollectReport(collector, result, excinfo) + +def pytest_runtest_protocol(item): + runtestprotocol(item) + return True + +def runtestprotocol(item, log=True): + rep = call_and_report(item, "setup", log) + reports = [rep] + if rep.passed: + reports.append(call_and_report(item, "call", log)) + reports.append(call_and_report(item, "teardown", log)) + return reports + +def pytest_runtest_setup(item): + item.config._setupstate.prepare(item) + +def pytest_runtest_call(item): + if not item._deprecated_testexecution(): + item.runtest() + +def pytest_runtest_makereport(item, call): + return ItemTestReport(item, call.excinfo, call.when) + +def pytest_runtest_teardown(item): + item.config._setupstate.teardown_exact(item) + +def pytest__teardown_final(session): + call = CallInfo(session.config._setupstate.teardown_all, when="teardown") + if call.excinfo: + ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir) + call.excinfo.traceback = ntraceback.filter() + rep = TeardownErrorReport(call.excinfo) + return rep + +def pytest_report_teststatus(report): + if report.when in ("setup", "teardown"): + if report.failed: + # category, shortletter, verbose-word + return "error", "E", "ERROR" + elif report.skipped: + return "skipped", "s", "SKIPPED" + else: + return "", "", "" +# +# Implementation + +def call_and_report(item, when, log=True): + call = call_runtest_hook(item, when) + hook = item.ihook + report = hook.pytest_runtest_makereport(item=item, call=call) + if log and (when == "call" or not report.passed): + hook.pytest_runtest_logreport(report=report) + return report + +def call_runtest_hook(item, when): + hookname = "pytest_runtest_" + when + ihook = getattr(item.ihook, hookname) + return CallInfo(lambda: ihook(item=item), when=when) + +class CallInfo: + excinfo = None + def __init__(self, func, when): + self.when = when + try: + self.result = func() + except KeyboardInterrupt: + raise + except: + self.excinfo = py.code.ExceptionInfo() + + def __repr__(self): + if self.excinfo: + status = "exception: %s" % str(self.excinfo.value) + else: + status = "result: %r" % (self.result,) + return "" % (self.when, status) + +class BaseReport(object): + def __repr__(self): + l = ["%s=%s" %(key, value) + for key, value in self.__dict__.items()] + return "<%s %s>" %(self.__class__.__name__, " ".join(l),) + + def toterminal(self, out): + longrepr = self.longrepr + if hasattr(longrepr, 'toterminal'): + longrepr.toterminal(out) + else: + out.line(str(longrepr)) + +class ItemTestReport(BaseReport): + failed = passed = skipped = False + + def __init__(self, item, excinfo=None, when=None): + self.item = item + self.when = when + if item and when != "setup": + self.keywords = item.readkeywords() + else: + # if we fail during setup it might mean + # we are not able to access the underlying object + # this might e.g. happen if we are unpickled + # and our parent collector did not collect us + # (because it e.g. skipped for platform reasons) + self.keywords = {} + if not excinfo: + self.passed = True + self.shortrepr = "." + else: + if not isinstance(excinfo, py.code.ExceptionInfo): + self.failed = True + shortrepr = "?" + longrepr = excinfo + elif excinfo.errisinstance(py.test.skip.Exception): + self.skipped = True + shortrepr = "s" + longrepr = self.item._repr_failure_py(excinfo) + else: + self.failed = True + shortrepr = self.item.shortfailurerepr + if self.when == "call": + longrepr = self.item.repr_failure(excinfo) + else: # exception in setup or teardown + longrepr = self.item._repr_failure_py(excinfo) + shortrepr = shortrepr.lower() + self.shortrepr = shortrepr + self.longrepr = longrepr + + def __repr__(self): + status = (self.passed and "passed" or + self.skipped and "skipped" or + self.failed and "failed" or + "CORRUPT") + l = [repr(self.item.name), "when=%r" % self.when, "outcome %r" % status,] + if hasattr(self, 'node'): + l.append("txnode=%s" % self.node.gateway.id) + info = " " .join(map(str, l)) + return "" % info + + def getnode(self): + return self.item + +class CollectReport(BaseReport): + skipped = failed = passed = False + + def __init__(self, collector, result, excinfo=None): + self.collector = collector + if not excinfo: + self.passed = True + self.result = result + else: + self.longrepr = self.collector._repr_failure_py(excinfo) + if excinfo.errisinstance(py.test.skip.Exception): + self.skipped = True + self.reason = str(excinfo.value) + else: + self.failed = True + + def getnode(self): + return self.collector + +class TeardownErrorReport(BaseReport): + skipped = passed = False + failed = True + when = "teardown" + def __init__(self, excinfo): + self.longrepr = excinfo.getrepr(funcargs=True) + +class SetupState(object): + """ shared state for setting up/tearing down test items or collectors. """ + def __init__(self): + self.stack = [] + self._finalizers = {} + + def addfinalizer(self, finalizer, colitem): + """ attach a finalizer to the given colitem. + if colitem is None, this will add a finalizer that + is called at the end of teardown_all(). + """ + assert hasattr(finalizer, '__call__') + #assert colitem in self.stack + self._finalizers.setdefault(colitem, []).append(finalizer) + + def _pop_and_teardown(self): + colitem = self.stack.pop() + self._teardown_with_finalization(colitem) + + def _callfinalizers(self, colitem): + finalizers = self._finalizers.pop(colitem, None) + while finalizers: + fin = finalizers.pop() + fin() + + def _teardown_with_finalization(self, colitem): + self._callfinalizers(colitem) + if colitem: + colitem.teardown() + for colitem in self._finalizers: + assert colitem is None or colitem in self.stack + + def teardown_all(self): + while self.stack: + self._pop_and_teardown() + self._teardown_with_finalization(None) + assert not self._finalizers + + def teardown_exact(self, item): + if self.stack and item == self.stack[-1]: + self._pop_and_teardown() + else: + self._callfinalizers(item) + + def prepare(self, colitem): + """ setup objects along the collector chain to the test-method + and teardown previously setup objects.""" + needed_collectors = colitem.listchain() + while self.stack: + if self.stack == needed_collectors[:len(self.stack)]: + break + self._pop_and_teardown() + # check if the last collection node has raised an error + for col in self.stack: + if hasattr(col, '_prepare_exc'): + py.builtin._reraise(*col._prepare_exc) + for col in needed_collectors[len(self.stack):]: + self.stack.append(col) + try: + col.setup() + except Exception: + col._prepare_exc = sys.exc_info() + raise + +# ============================================================= +# Test OutcomeExceptions and helpers for creating them. + + +class OutcomeException(Exception): + """ OutcomeException and its subclass instances indicate and + contain info about test and collection outcomes. + """ + def __init__(self, msg=None, excinfo=None): + self.msg = msg + self.excinfo = excinfo + + def __repr__(self): + if self.msg: + return repr(self.msg) + return "<%s instance>" %(self.__class__.__name__,) + __str__ = __repr__ + +class Skipped(OutcomeException): + # XXX hackish: on 3k we fake to live in the builtins + # in order to have Skipped exception printing shorter/nicer + __module__ = 'builtins' + +class Failed(OutcomeException): + """ raised from an explicit call to py.test.fail() """ + __module__ = 'builtins' + +class ExceptionFailure(Failed): + """ raised by py.test.raises on an exception-assertion mismatch. """ + def __init__(self, expr, expected, msg=None, excinfo=None): + Failed.__init__(self, msg=msg, excinfo=excinfo) + self.expr = expr + self.expected = expected + +class Exit(KeyboardInterrupt): + """ raised by py.test.exit for immediate program exits without tracebacks and reporter/summary. """ + def __init__(self, msg="unknown reason"): + self.msg = msg + KeyboardInterrupt.__init__(self, msg) + +# exposed helper methods + +def exit(msg): + """ exit testing process as if KeyboardInterrupt was triggered. """ + __tracebackhide__ = True + raise Exit(msg) + +exit.Exception = Exit + +def skip(msg=""): + """ skip an executing test with the given message. Note: it's usually + better use the py.test.mark.skipif marker to declare a test to be + skipped under certain conditions like mismatching platforms or + dependencies. See the pytest_skipping plugin for details. + """ + __tracebackhide__ = True + raise Skipped(msg=msg) + +skip.Exception = Skipped + +def fail(msg=""): + """ explicitely fail an currently-executing test with the given Message. """ + __tracebackhide__ = True + raise Failed(msg=msg) + +fail.Exception = Failed + +def raises(ExpectedException, *args, **kwargs): + """ if args[0] is callable: raise AssertionError if calling it with + the remaining arguments does not raise the expected exception. + if args[0] is a string: raise AssertionError if executing the + the string in the calling scope does not raise expected exception. + for examples: + x = 5 + raises(TypeError, lambda x: x + 'hello', x=x) + raises(TypeError, "x + 'hello'") + """ + __tracebackhide__ = True + assert args + if isinstance(args[0], str): + code, = args + assert isinstance(code, str) + frame = sys._getframe(1) + loc = frame.f_locals.copy() + loc.update(kwargs) + #print "raises frame scope: %r" % frame.f_locals + try: + code = py.code.Source(code).compile() + py.builtin.exec_(code, frame.f_globals, loc) + # XXX didn'T mean f_globals == f_locals something special? + # this is destroyed here ... + except ExpectedException: + return py.code.ExceptionInfo() + else: + func = args[0] + try: + func(*args[1:], **kwargs) + except ExpectedException: + return py.code.ExceptionInfo() + k = ", ".join(["%s=%r" % x for x in kwargs.items()]) + if k: + k = ', ' + k + expr = '%s(%r%s)' %(getattr(func, '__name__', func), args, k) + raise ExceptionFailure(msg="DID NOT RAISE", + expr=args, expected=ExpectedException) + +raises.Exception = ExceptionFailure + +def importorskip(modname, minversion=None): + """ return imported module if it has a higher __version__ than the + optionally specified 'minversion' - otherwise call py.test.skip() + with a message detailing the mismatch. + """ + compile(modname, '', 'eval') # to catch syntaxerrors + try: + mod = __import__(modname, None, None, ['__doc__']) + except ImportError: + py.test.skip("could not import %r" %(modname,)) + if minversion is None: + return mod + verattr = getattr(mod, '__version__', None) + if isinstance(minversion, str): + minver = minversion.split(".") + else: + minver = list(minversion) + if verattr is None or verattr.split(".") < minver: + py.test.skip("module %r has __version__ %r, required is: %r" %( + modname, verattr, minversion)) + return mod + Added: pypy/branch/py12/py/_plugin/pytest_skipping.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_skipping.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,243 @@ +""" +advanced skipping for python test functions, classes or modules. + +With this plugin you can mark test functions for conditional skipping +or as "xfail", expected-to-fail. Skipping a test will avoid running it +while xfail-marked tests will run and result in an inverted outcome: +a pass becomes a failure and a fail becomes a semi-passing one. + +The need for skipping a test is usually connected to a condition. +If a test fails under all conditions then it's probably better +to mark your test as 'xfail'. + +By passing ``--report=xfailed,skipped`` to the terminal reporter +you will see summary information on skips and xfail-run tests +at the end of a test run. + +.. _skipif: + +Skipping a single function +------------------------------------------- + +Here is an example for marking a test function to be skipped +when run on a Python3 interpreter:: + + @py.test.mark.skipif("sys.version_info >= (3,0)") + def test_function(): + ... + +During test function setup the skipif condition is +evaluated by calling ``eval(expr, namespace)``. The namespace +contains the ``sys`` and ``os`` modules and the test +``config`` object. The latter allows you to skip based +on a test configuration value e.g. like this:: + + @py.test.mark.skipif("not config.getvalue('db')") + def test_function(...): + ... + +Create a shortcut for your conditional skip decorator +at module level like this:: + + win32only = py.test.mark.skipif("sys.platform != 'win32'") + + @win32only + def test_function(): + ... + + +skip groups of test functions +-------------------------------------- + +As with all metadata function marking you can do it at +`whole class- or module level`_. Here is an example +for skipping all methods of a test class based on platform:: + + class TestPosixCalls: + pytestmark = py.test.mark.skipif("sys.platform == 'win32'") + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +The ``pytestmark`` decorator will be applied to each test function. + +.. _`whole class- or module level`: mark.html#scoped-marking + + +mark a test function as **expected to fail** +------------------------------------------------------- + +You can use the ``xfail`` marker to indicate that you +expect the test to fail:: + + @py.test.mark.xfail + def test_function(): + ... + +This test will be run but no traceback will be reported +when it fails. Instead terminal reporting will list it in the +"expected to fail" or "unexpectedly passing" sections. + +Same as with skipif_ you can also selectively expect a failure +depending on platform:: + + @py.test.mark.xfail("sys.version_info >= (3,0)") + + def test_function(): + ... + + +skipping on a missing import dependency +-------------------------------------------------- + +You can use the following import helper at module level +or within a test or test setup function:: + + docutils = py.test.importorskip("docutils") + +If ``docutils`` cannot be imported here, this will lead to a +skip outcome of the test. You can also skip dependeing if +if a library does not come with a high enough version:: + + docutils = py.test.importorskip("docutils", minversion="0.3") + +The version will be read from the specified module's ``__version__`` attribute. + +imperative skip from within a test or setup function +------------------------------------------------------ + +If for some reason you cannot declare skip-conditions +you can also imperatively produce a Skip-outcome from +within test or setup code. Example:: + + def test_function(): + if not valid_config(): + py.test.skip("unsuppored configuration") + +""" +# XXX py.test.skip, .importorskip and the Skipped class +# should also be defined in this plugin, requires thought/changes + +import py + + +def pytest_runtest_setup(item): + expr, result = evalexpression(item, 'skipif') + if result: + py.test.skip(expr) + +def pytest_runtest_makereport(__multicall__, item, call): + if call.when != "call": + return + expr, result = evalexpression(item, 'xfail') + rep = __multicall__.execute() + if result: + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = expr + else: + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep + +# called by terminalreporter progress reporting +def pytest_report_teststatus(report): + if 'xfail' in report.keywords: + if report.skipped: + return "xfailed", "x", "xfail" + elif report.failed: + return "xpassed", "P", "xpass" + +# called by the terminalreporter instance/plugin +def pytest_terminal_summary(terminalreporter): + show_xfailed(terminalreporter) + show_skipped(terminalreporter) + +def show_xfailed(terminalreporter): + tr = terminalreporter + xfailed = tr.stats.get("xfailed") + if xfailed: + if not tr.hasopt('xfailed'): + tr.write_line( + "%d expected failures, use --report=xfailed for more info" % + len(xfailed)) + return + tr.write_sep("_", "expected failures") + for rep in xfailed: + entry = rep.longrepr.reprcrash + modpath = rep.item.getmodpath(includemodule=True) + pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) + reason = rep.longrepr.reprcrash.message + i = reason.find("\n") + if i != -1: + reason = reason[:i] + tr._tw.line("%s %s" %(pos, reason)) + + xpassed = terminalreporter.stats.get("xpassed") + if xpassed: + tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") + for rep in xpassed: + fspath, lineno, modpath = rep.item.reportinfo() + pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) + tr._tw.line(pos) + + +def evalexpression(item, keyword): + if isinstance(item, py.test.collect.Function): + markholder = getattr(item.obj, keyword, None) + result = False + if markholder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} + expr, result = None, True + for expr in markholder.args: + if isinstance(expr, str): + result = cached_eval(item.config, expr, d) + else: + result = expr + if not result: + break + return expr, result + return None, False + +def cached_eval(config, expr, d): + if not hasattr(config, '_evalcache'): + config._evalcache = {} + try: + return config._evalcache[expr] + except KeyError: + #import sys + #print >>sys.stderr, ("cache-miss: %r" % expr) + config._evalcache[expr] = x = eval(expr, d) + return x + + +def folded_skips(skipped): + d = {} + for event in skipped: + entry = event.longrepr.reprcrash + key = entry.path, entry.lineno, entry.message + d.setdefault(key, []).append(event) + l = [] + for key, events in d.items(): + l.append((len(events),) + key) + return l + +def show_skipped(terminalreporter): + tr = terminalreporter + skipped = tr.stats.get('skipped', []) + if skipped: + if not tr.hasopt('skipped'): + tr.write_line( + "%d skipped tests, use --report=skipped for more info" % + len(skipped)) + return + fskips = folded_skips(skipped) + if fskips: + tr.write_sep("_", "skipped test summary") + for num, fspath, lineno, reason in fskips: + tr._tw.line("%s:%d: [%d] %s" %(fspath, lineno, num, reason)) Added: pypy/branch/py12/py/_plugin/pytest_terminal.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_terminal.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,511 @@ +""" +Implements terminal reporting of the full testing process. + +This is a good source for looking at the various reporting hooks. +""" +import py +import sys + +optionalhook = py.test.mark.optionalhook + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", "reporting", after="general") + group._addoption('-v', '--verbose', action="count", + dest="verbose", default=0, help="increase verbosity."), + group._addoption('-l', '--showlocals', + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group.addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="show more info, valid: skipped,xfailed") + group._addoption('--tb', metavar="style", + action="store", dest="tbstyle", default='long', + type="choice", choices=['long', 'short', 'no', 'line'], + help="traceback print mode (long/short/line/no).") + group._addoption('--fulltrace', + action="store_true", dest="fulltrace", default=False, + help="don't cut any tracebacks (default is to cut).") + group._addoption('--funcargs', + action="store_true", dest="showfuncargs", default=False, + help="show available function arguments, sorted by plugin") + +def pytest_configure(config): + if config.option.collectonly: + reporter = CollectonlyReporter(config) + elif config.option.showfuncargs: + config.setsessionclass(ShowFuncargSession) + reporter = None + else: + reporter = TerminalReporter(config) + if reporter: + # XXX see remote.py's XXX + for attr in 'pytest_terminal_hasmarkup', 'pytest_terminal_fullwidth': + if hasattr(config, attr): + #print "SETTING TERMINAL OPTIONS", attr, getattr(config, attr) + name = attr.split("_")[-1] + assert hasattr(self.reporter._tw, name), name + setattr(reporter._tw, name, getattr(config, attr)) + config.pluginmanager.register(reporter, 'terminalreporter') + +def getreportopt(optvalue): + d = {} + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + val = True + if setting.startswith("no"): + val = False + setting = setting[2:] + d[setting] = val + return d + +class TerminalReporter: + def __init__(self, config, file=None): + self.config = config + self.stats = {} + self.curdir = py.path.local() + if file is None: + file = py.std.sys.stdout + self._tw = py.io.TerminalWriter(file) + self.currentfspath = None + self.gateway2info = {} + self._reportopt = getreportopt(config.getvalue('report')) + + def hasopt(self, name): + return self._reportopt.get(name, False) + + def write_fspath_result(self, fspath, res): + fspath = self.curdir.bestrelpath(fspath) + if fspath != self.currentfspath: + self._tw.line() + relpath = self.curdir.bestrelpath(fspath) + self._tw.write(relpath + " ") + self.currentfspath = fspath + self._tw.write(res) + + def write_ensure_prefix(self, prefix, extra="", **kwargs): + if self.currentfspath != prefix: + self._tw.line() + self.currentfspath = prefix + self._tw.write(prefix) + if extra: + self._tw.write(extra, **kwargs) + self.currentfspath = -2 + + def ensure_newline(self): + if self.currentfspath: + self._tw.line() + self.currentfspath = None + + def write_line(self, line, **markup): + line = str(line) + self.ensure_newline() + self._tw.line(line, **markup) + + def write_sep(self, sep, title=None, **markup): + self.ensure_newline() + self._tw.sep(sep, title, **markup) + + def getcategoryletterword(self, rep): + res = self.config.hook.pytest_report_teststatus(report=rep) + if res: + return res + for cat in 'skipped failed passed ???'.split(): + if getattr(rep, cat, None): + break + return cat, self.getoutcomeletter(rep), self.getoutcomeword(rep) + + def getoutcomeletter(self, rep): + return rep.shortrepr + + def getoutcomeword(self, rep): + if rep.passed: + return "PASS", dict(green=True) + elif rep.failed: + return "FAIL", dict(red=True) + elif rep.skipped: + return "SKIP" + else: + return "???", dict(red=True) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.write_line("INTERNALERROR> " + line) + + def pytest_plugin_registered(self, plugin): + if self.config.option.traceconfig: + msg = "PLUGIN registered: %s" %(plugin,) + # XXX this event may happen during setup/teardown time + # which unfortunately captures our output here + # which garbles our output if we use self.write_line + self.write_line(msg) + + @optionalhook + def pytest_gwmanage_newgateway(self, gateway, platinfo): + #self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec)) + d = {} + d['version'] = repr_pythonversion(platinfo.version_info) + d['id'] = gateway.id + d['spec'] = gateway.spec._spec + d['platform'] = platinfo.platform + if self.config.option.verbose: + d['extra'] = "- " + platinfo.executable + else: + d['extra'] = "" + d['cwd'] = platinfo.cwd + infoline = ("[%(id)s] %(spec)s -- platform %(platform)s, " + "Python %(version)s " + "cwd: %(cwd)s" + "%(extra)s" % d) + self.write_line(infoline) + self.gateway2info[gateway] = infoline + + @optionalhook + def pytest_testnodeready(self, node): + self.write_line("[%s] txnode ready to receive tests" %(node.gateway.id,)) + + @optionalhook + def pytest_testnodedown(self, node, error): + if error: + self.write_line("[%s] node down, error: %s" %(node.gateway.id, error)) + + @optionalhook + def pytest_rescheduleitems(self, items): + if self.config.option.debug: + self.write_sep("!", "RESCHEDULING %s " %(items,)) + + @optionalhook + def pytest_looponfailinfo(self, failreports, rootdirs): + if failreports: + self.write_sep("#", "LOOPONFAILING", red=True) + for report in failreports: + loc = self._getcrashline(report) + self.write_line(loc, red=True) + self.write_sep("#", "waiting for changes") + for rootdir in rootdirs: + self.write_line("### Watching: %s" %(rootdir,), bold=True) + + + def pytest_trace(self, category, msg): + if self.config.option.debug or \ + self.config.option.traceconfig and category.find("config") != -1: + self.write_line("[%s] %s" %(category, msg)) + + def pytest_deselected(self, items): + self.stats.setdefault('deselected', []).append(items) + + def pytest_itemstart(self, item, node=None): + if getattr(self.config.option, 'dist', 'no') != "no": + # for dist-testing situations itemstart means we + # queued the item for sending, not interesting (unless debugging) + if self.config.option.debug: + line = self._reportinfoline(item) + extra = "" + if node: + extra = "-> [%s]" % node.gateway.id + self.write_ensure_prefix(line, extra) + else: + if self.config.option.verbose: + line = self._reportinfoline(item) + self.write_ensure_prefix(line, "") + else: + # ensure that the path is printed before the + # 1st test of a module starts running + + self.write_fspath_result(self._getfspath(item), "") + + def pytest__teardown_final_logerror(self, report): + self.stats.setdefault("error", []).append(report) + + def pytest_runtest_logreport(self, report): + rep = report + cat, letter, word = self.getcategoryletterword(rep) + if not letter and not word: + # probably passed setup/teardown + return + if isinstance(word, tuple): + word, markup = word + else: + markup = {} + self.stats.setdefault(cat, []).append(rep) + if not self.config.option.verbose: + self.write_fspath_result(self._getfspath(rep.item), letter) + else: + line = self._reportinfoline(rep.item) + if not hasattr(rep, 'node'): + self.write_ensure_prefix(line, word, **markup) + else: + self.ensure_newline() + if hasattr(rep, 'node'): + self._tw.write("[%s] " % rep.node.gateway.id) + self._tw.write(word, **markup) + self._tw.write(" " + line) + self.currentfspath = -2 + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.stats.setdefault("error", []).append(report) + msg = report.longrepr.reprcrash.message + self.write_fspath_result(report.collector.fspath, "E") + elif report.skipped: + self.stats.setdefault("skipped", []).append(report) + self.write_fspath_result(report.collector.fspath, "S") + + def pytest_sessionstart(self, session): + self.write_sep("=", "test session starts", bold=True) + self._sessionstarttime = py.std.time.time() + + verinfo = ".".join(map(str, sys.version_info[:3])) + msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg += " -- pytest-%s" % (py.__version__) + if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): + msg += " -- " + str(sys.executable) + self.write_line(msg) + lines = self.config.hook.pytest_report_header(config=self.config) + lines.reverse() + for line in flatten(lines): + self.write_line(line) + for i, testarg in enumerate(self.config.args): + self.write_line("test object %d: %s" %(i+1, testarg)) + + def pytest_sessionfinish(self, exitstatus, __multicall__): + __multicall__.execute() + self._tw.line("") + if exitstatus in (0, 1, 2): + self.summary_errors() + self.summary_failures() + self.config.hook.pytest_terminal_summary(terminalreporter=self) + if exitstatus == 2: + self._report_keyboardinterrupt() + self.summary_deselected() + self.summary_stats() + + def pytest_keyboard_interrupt(self, excinfo): + self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True) + + def _report_keyboardinterrupt(self): + self.write_sep("!", "KEYBOARD INTERRUPT") + excrepr = self._keyboardinterrupt_memo + if self.config.option.verbose: + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) + + def _getcrashline(self, report): + try: + return report.longrepr.reprcrash + except AttributeError: + return str(report.longrepr)[:50] + + def _reportinfoline(self, item): + collect_fspath = self._getfspath(item) + fspath, lineno, msg = self._getreportinfo(item) + if fspath and fspath != collect_fspath: + fspath = "%s <- %s" % ( + self.curdir.bestrelpath(collect_fspath), + self.curdir.bestrelpath(fspath)) + elif fspath: + fspath = self.curdir.bestrelpath(fspath) + if lineno is not None: + lineno += 1 + if fspath and lineno and msg: + line = "%(fspath)s:%(lineno)s: %(msg)s" + elif fspath and msg: + line = "%(fspath)s: %(msg)s" + elif fspath and lineno: + line = "%(fspath)s:%(lineno)s %(extrapath)s" + else: + line = "[noreportinfo]" + return line % locals() + " " + + def _getfailureheadline(self, rep): + if hasattr(rep, "collector"): + return str(rep.collector.fspath) + elif hasattr(rep, 'item'): + fspath, lineno, msg = self._getreportinfo(rep.item) + return msg + else: + return "test session" + + def _getreportinfo(self, item): + try: + return item.__reportinfo + except AttributeError: + pass + reportinfo = item.config.hook.pytest_report_iteminfo(item=item) + # cache on item + item.__reportinfo = reportinfo + return reportinfo + + def _getfspath(self, item): + try: + return item.fspath + except AttributeError: + fspath, lineno, msg = self._getreportinfo(item) + return fspath + + # + # summaries for sessionfinish + # + + def summary_failures(self): + tbstyle = self.config.getvalue("tbstyle") + if 'failed' in self.stats and tbstyle != "no": + self.write_sep("=", "FAILURES") + for rep in self.stats['failed']: + if tbstyle == "line": + line = self._getcrashline(rep) + self.write_line(line) + else: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def summary_errors(self): + if 'error' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "ERRORS") + for rep in self.stats['error']: + msg = self._getfailureheadline(rep) + if not hasattr(rep, 'when'): + # collect + msg = "ERROR during collection " + msg + elif rep.when == "setup": + msg = "ERROR at setup of " + msg + elif rep.when == "teardown": + msg = "ERROR at teardown of " + msg + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def write_platinfo(self, rep): + if hasattr(rep, 'node'): + self.write_line(self.gateway2info.get( + rep.node.gateway, + "node %r (platinfo not found? strange)") + [:self._tw.fullwidth-1]) + + def summary_stats(self): + session_duration = py.std.time.time() - self._sessionstarttime + + keys = "failed passed skipped deselected".split() + for key in self.stats.keys(): + if key not in keys: + keys.append(key) + parts = [] + for key in keys: + val = self.stats.get(key, None) + if val: + parts.append("%d %s" %(len(val), key)) + line = ", ".join(parts) + # XXX coloring + self.write_sep("=", "%s in %.2f seconds" %(line, session_duration)) + + def summary_deselected(self): + if 'deselected' in self.stats: + self.write_sep("=", "%d tests deselected by %r" %( + len(self.stats['deselected']), self.config.option.keyword), bold=True) + + +class CollectonlyReporter: + INDENT = " " + + def __init__(self, config, out=None): + self.config = config + if out is None: + out = py.std.sys.stdout + self.out = py.io.TerminalWriter(out) + self.indent = "" + self._failed = [] + + def outindent(self, line): + self.out.line(self.indent + str(line)) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.out.line("INTERNALERROR> " + line) + + def pytest_collectstart(self, collector): + self.outindent(collector) + self.indent += self.INDENT + + def pytest_itemstart(self, item, node=None): + self.outindent(item) + + def pytest_collectreport(self, report): + if not report.passed: + self.outindent("!!! %s !!!" % report.longrepr.reprcrash.message) + self._failed.append(report) + self.indent = self.indent[:-len(self.INDENT)] + + def pytest_sessionfinish(self, session, exitstatus): + if self._failed: + self.out.sep("!", "collection failures") + for rep in self._failed: + rep.toterminal(self.out) + + +def repr_pythonversion(v=None): + if v is None: + v = sys.version_info + try: + return "%s.%s.%s-%s-%s" % v + except (TypeError, ValueError): + return str(v) + +def flatten(l): + for x in l: + if isinstance(x, (list, tuple)): + for y in flatten(x): + yield y + else: + yield x + +from py._test.session import Session +class ShowFuncargSession(Session): + def main(self, colitems): + self.fspath = py.path.local() + self.sessionstarts() + try: + self.showargs(colitems[0]) + finally: + self.sessionfinishes(exitstatus=1) + + def showargs(self, colitem): + tw = py.io.TerminalWriter() + from py._test.funcargs import getplugins + from py._test.funcargs import FuncargRequest + plugins = getplugins(colitem, withpy=True) + verbose = self.config.getvalue("verbose") + for plugin in plugins: + available = [] + for name, factory in vars(plugin).items(): + if name.startswith(FuncargRequest._argprefix): + name = name[len(FuncargRequest._argprefix):] + if name not in available: + available.append([name, factory]) + if available: + pluginname = plugin.__name__ + for name, factory in available: + loc = self.getlocation(factory) + if verbose: + funcargspec = "%s -- %s" %(name, loc,) + else: + funcargspec = name + tw.line(funcargspec, green=True) + doc = factory.__doc__ or "" + if doc: + for line in doc.split("\n"): + tw.line(" " + line.strip()) + else: + tw.line(" %s: no docstring available" %(loc,), + red=True) + + def getlocation(self, function): + import inspect + fn = py.path.local(inspect.getfile(function)) + lineno = py.builtin._getcode(function).co_firstlineno + if fn.relto(self.fspath): + fn = fn.relto(self.fspath) + return "%s:%d" %(fn, lineno+1) Added: pypy/branch/py12/py/_plugin/pytest_terminal.py.orig ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_terminal.py.orig Fri Apr 30 17:07:52 2010 @@ -0,0 +1,513 @@ +""" +Implements terminal reporting of the full testing process. + +This is a good source for looking at the various reporting hooks. +""" +import py +import sys + +optionalhook = py.test.mark.optionalhook + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", "reporting", after="general") + group._addoption('-v', '--verbose', action="count", + dest="verbose", default=0, help="increase verbosity."), + group._addoption('-l', '--showlocals', + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group.addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="show more info, valid: skipped,xfailed") + group._addoption('--tb', metavar="style", + action="store", dest="tbstyle", default='long', + type="choice", choices=['long', 'short', 'no', 'line'], + help="traceback print mode (long/short/line/no).") + group._addoption('--fulltrace', + action="store_true", dest="fulltrace", default=False, + help="don't cut any tracebacks (default is to cut).") + group._addoption('--funcargs', + action="store_true", dest="showfuncargs", default=False, + help="show available function arguments, sorted by plugin") + +def pytest_configure(config): + if config.option.collectonly: + reporter = CollectonlyReporter(config) + elif config.option.showfuncargs: + config.setsessionclass(ShowFuncargSession) + reporter = None + else: + reporter = TerminalReporter(config) + if reporter: + # XXX see remote.py's XXX + for attr in 'pytest_terminal_hasmarkup', 'pytest_terminal_fullwidth': + if hasattr(config, attr): + #print "SETTING TERMINAL OPTIONS", attr, getattr(config, attr) + name = attr.split("_")[-1] + assert hasattr(self.reporter._tw, name), name + setattr(reporter._tw, name, getattr(config, attr)) + config.pluginmanager.register(reporter, 'terminalreporter') + +def getreportopt(optvalue): + d = {} + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + val = True + if setting.startswith("no"): + val = False + setting = setting[2:] + d[setting] = val + return d + +class TerminalReporter: + def __init__(self, config, file=None): + self.config = config + self.stats = {} + self.curdir = py.path.local() + if file is None: + file = py.std.sys.stdout + self._tw = py.io.TerminalWriter(file) + self.currentfspath = None + self.gateway2info = {} + self._reportopt = getreportopt(config.getvalue('report')) + + def hasopt(self, name): + return self._reportopt.get(name, False) + + def write_fspath_result(self, fspath, res): + fspath = self.curdir.bestrelpath(fspath) + if fspath != self.currentfspath: + self._tw.line() + relpath = self.curdir.bestrelpath(fspath) + self._tw.write(relpath + " ") + self.currentfspath = fspath + self._tw.write(res) + + def write_ensure_prefix(self, prefix, extra="", **kwargs): + if self.currentfspath != prefix: + self._tw.line() + self.currentfspath = prefix + self._tw.write(prefix) + if extra: + self._tw.write(extra, **kwargs) + self.currentfspath = -2 + + def ensure_newline(self): + if self.currentfspath: + self._tw.line() + self.currentfspath = None + + def write_line(self, line, **markup): + line = str(line) + self.ensure_newline() + self._tw.line(line, **markup) + + def write_sep(self, sep, title=None, **markup): + self.ensure_newline() + self._tw.sep(sep, title, **markup) + + def getcategoryletterword(self, rep): + res = self.config.hook.pytest_report_teststatus(report=rep) + if res: + return res + for cat in 'skipped failed passed ???'.split(): + if getattr(rep, cat, None): + break + return cat, self.getoutcomeletter(rep), self.getoutcomeword(rep) + + def getoutcomeletter(self, rep): + return rep.shortrepr + + def getoutcomeword(self, rep): + if rep.passed: + return "PASS", dict(green=True) + elif rep.failed: + return "FAIL", dict(red=True) + elif rep.skipped: + return "SKIP" + else: + return "???", dict(red=True) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.write_line("INTERNALERROR> " + line) + + def pytest_plugin_registered(self, plugin): + if self.config.option.traceconfig: + msg = "PLUGIN registered: %s" %(plugin,) + # XXX this event may happen during setup/teardown time + # which unfortunately captures our output here + # which garbles our output if we use self.write_line + self.write_line(msg) + + @optionalhook + def pytest_gwmanage_newgateway(self, gateway, platinfo): + #self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec)) + d = {} + d['version'] = repr_pythonversion(platinfo.version_info) + d['id'] = gateway.id + d['spec'] = gateway.spec._spec + d['platform'] = platinfo.platform + if self.config.option.verbose: + d['extra'] = "- " + platinfo.executable + else: + d['extra'] = "" + d['cwd'] = platinfo.cwd + infoline = ("[%(id)s] %(spec)s -- platform %(platform)s, " + "Python %(version)s " + "cwd: %(cwd)s" + "%(extra)s" % d) + self.write_line(infoline) + self.gateway2info[gateway] = infoline + + @optionalhook + def pytest_testnodeready(self, node): + self.write_line("[%s] txnode ready to receive tests" %(node.gateway.id,)) + + @optionalhook + def pytest_testnodedown(self, node, error): + if error: + self.write_line("[%s] node down, error: %s" %(node.gateway.id, error)) + + @optionalhook + def pytest_rescheduleitems(self, items): + if self.config.option.debug: + self.write_sep("!", "RESCHEDULING %s " %(items,)) + + @optionalhook + def pytest_looponfailinfo(self, failreports, rootdirs): + if failreports: + self.write_sep("#", "LOOPONFAILING", red=True) + for report in failreports: + loc = self._getcrashline(report) + self.write_line(loc, red=True) + self.write_sep("#", "waiting for changes") + for rootdir in rootdirs: + self.write_line("### Watching: %s" %(rootdir,), bold=True) + + + def pytest_trace(self, category, msg): + if self.config.option.debug or \ + self.config.option.traceconfig and category.find("config") != -1: + self.write_line("[%s] %s" %(category, msg)) + + def pytest_deselected(self, items): + self.stats.setdefault('deselected', []).append(items) + + def pytest_itemstart(self, item, node=None): + if getattr(self.config.option, 'dist', 'no') != "no": + # for dist-testing situations itemstart means we + # queued the item for sending, not interesting (unless debugging) + if self.config.option.debug: + line = self._reportinfoline(item) + extra = "" + if node: + extra = "-> [%s]" % node.gateway.id + self.write_ensure_prefix(line, extra) + else: + if self.config.option.verbose: + line = self._reportinfoline(item) + self.write_ensure_prefix(line, "") + else: + # ensure that the path is printed before the + # 1st test of a module starts running + + self.write_fspath_result(self._getfspath(item), "") + + def pytest__teardown_final_logerror(self, report): + self.stats.setdefault("error", []).append(report) + + def pytest_runtest_logreport(self, report): + rep = report + cat, letter, word = self.getcategoryletterword(rep) + if not letter and not word: + # probably passed setup/teardown + return + if isinstance(word, tuple): + word, markup = word + else: + markup = {} + self.stats.setdefault(cat, []).append(rep) + if not self.config.option.verbose: + self.write_fspath_result(self._getfspath(rep.item), letter) + else: + line = self._reportinfoline(rep.item) + if not hasattr(rep, 'node'): + self.write_ensure_prefix(line, word, **markup) + else: + self.ensure_newline() + if hasattr(rep, 'node'): + self._tw.write("[%s] " % rep.node.gateway.id) + self._tw.write(word, **markup) + self._tw.write(" " + line) + self.currentfspath = -2 + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.stats.setdefault("error", []).append(report) + msg = report.longrepr.reprcrash.message + self.write_fspath_result(report.collector.fspath, "E") + elif report.skipped: + self.stats.setdefault("skipped", []).append(report) + self.write_fspath_result(report.collector.fspath, "S") + + def pytest_sessionstart(self, session): + self.write_sep("=", "test session starts", bold=True) + self._sessionstarttime = py.std.time.time() + + verinfo = ".".join(map(str, sys.version_info[:3])) + msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg += " -- pytest-%s" % (py.__version__) + if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): + msg += " -- " + str(sys.executable) + self.write_line(msg) + lines = self.config.hook.pytest_report_header(config=self.config) + lines.reverse() + for line in flatten(lines): + self.write_line(line) + for i, testarg in enumerate(self.config.args): + self.write_line("test object %d: %s" %(i+1, testarg)) + + def pytest_sessionfinish(self, exitstatus, __multicall__): + __multicall__.execute() + self._tw.line("") + if exitstatus in (0, 1, 2): + self.summary_errors() + self.summary_failures() + self.config.hook.pytest_terminal_summary(terminalreporter=self) + if exitstatus == 2: + self._report_keyboardinterrupt() + self.summary_deselected() + self.summary_stats() + + def pytest_keyboard_interrupt(self, excinfo): + self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True) + + def _report_keyboardinterrupt(self): + self.write_sep("!", "KEYBOARD INTERRUPT") + excrepr = self._keyboardinterrupt_memo + if self.config.option.verbose: + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) + + def _getcrashline(self, report): + try: + return report.longrepr.reprcrash + except AttributeError: + return str(report.longrepr)[:50] + + def _reportinfoline(self, item): + collect_fspath = self._getfspath(item) + fspath, lineno, msg = self._getreportinfo(item) + if fspath and fspath != collect_fspath: + fspath = "%s <- %s" % ( + self.curdir.bestrelpath(collect_fspath), + self.curdir.bestrelpath(fspath)) + elif fspath: + fspath = self.curdir.bestrelpath(fspath) + if lineno is not None: + lineno += 1 + if fspath and lineno and msg: + line = "%(fspath)s:%(lineno)s: %(msg)s" + elif fspath and msg: + line = "%(fspath)s: %(msg)s" + elif fspath and lineno: + line = "%(fspath)s:%(lineno)s %(extrapath)s" + else: + line = "[noreportinfo]" + return line % locals() + " " + + def _getfailureheadline(self, rep): + if hasattr(rep, "collector"): + return str(rep.collector.fspath) + elif hasattr(rep, 'item'): + fspath, lineno, msg = self._getreportinfo(rep.item) + return msg + else: + return "test session" + + def _getreportinfo(self, item): + try: + return item.__reportinfo + except AttributeError: + pass + reportinfo = item.config.hook.pytest_report_iteminfo(item=item) + # cache on item + item.__reportinfo = reportinfo + return reportinfo + + def _getfspath(self, item): + try: + return item.fspath + except AttributeError: + fspath, lineno, msg = self._getreportinfo(item) + return fspath + + # + # summaries for sessionfinish + # + + def summary_failures(self): + tbstyle = self.config.getvalue("tbstyle") + if 'failed' in self.stats and tbstyle != "no": + self.write_sep("=", "FAILURES") + for rep in self.stats['failed']: + if tbstyle == "line": + line = self._getcrashline(rep) + self.write_line(line) + else: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def summary_errors(self): + if 'error' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "ERRORS") + for rep in self.stats['error']: + msg = self._getfailureheadline(rep) + if not hasattr(rep, 'when'): + # collect + msg = "ERROR during collection " + msg + elif rep.when == "setup": + msg = "ERROR at setup of " + msg + elif rep.when == "teardown": + msg = "ERROR at teardown of " + msg + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def write_platinfo(self, rep): + if hasattr(rep, 'node'): + self.write_line(self.gateway2info.get( + rep.node.gateway, + "node %r (platinfo not found? strange)") + [:self._tw.fullwidth-1]) + + def summary_stats(self): + session_duration = py.std.time.time() - self._sessionstarttime + + keys = "failed passed skipped deselected".split() + for key in self.stats.keys(): + if key not in keys: + keys.append(key) + parts = [] + for key in keys: + val = self.stats.get(key, None) + if val: + parts.append("%d %s" %(len(val), key)) + line = ", ".join(parts) + # XXX coloring + self.write_sep("=", "%s in %.2f seconds" %(line, session_duration)) + + def summary_deselected(self): + if 'deselected' in self.stats: + self.write_sep("=", "%d tests deselected by %r" %( + len(self.stats['deselected']), self.config.option.keyword), bold=True) + + +class CollectonlyReporter: + INDENT = " " + + def __init__(self, config, out=None): + self.config = config + if out is None: + out = py.std.sys.stdout + self.out = py.io.TerminalWriter(out) + self.indent = "" + self._failed = [] + + def outindent(self, line): + s = self.indent + str(line) + print ("printing: %s" % s) + self.out.line(self.indent + str(line)) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.out.line("INTERNALERROR> " + line) + + def pytest_collectstart(self, collector): + self.outindent(collector) + self.indent += self.INDENT + + def pytest_itemstart(self, item, node=None): + self.outindent(item) + + def pytest_collectreport(self, report): + if not report.passed: + self.outindent("!!! %s !!!" % report.longrepr.reprcrash.message) + self._failed.append(report) + self.indent = self.indent[:-len(self.INDENT)] + + def pytest_sessionfinish(self, session, exitstatus): + if self._failed: + self.out.sep("!", "collection failures") + for rep in self._failed: + rep.toterminal(self.out) + + +def repr_pythonversion(v=None): + if v is None: + v = sys.version_info + try: + return "%s.%s.%s-%s-%s" % v + except (TypeError, ValueError): + return str(v) + +def flatten(l): + for x in l: + if isinstance(x, (list, tuple)): + for y in flatten(x): + yield y + else: + yield x + +from py._test.session import Session +class ShowFuncargSession(Session): + def main(self, colitems): + self.fspath = py.path.local() + self.sessionstarts() + try: + self.showargs(colitems[0]) + finally: + self.sessionfinishes(exitstatus=1) + + def showargs(self, colitem): + tw = py.io.TerminalWriter() + from py._test.funcargs import getplugins + from py._test.funcargs import FuncargRequest + plugins = getplugins(colitem, withpy=True) + verbose = self.config.getvalue("verbose") + for plugin in plugins: + available = [] + for name, factory in vars(plugin).items(): + if name.startswith(FuncargRequest._argprefix): + name = name[len(FuncargRequest._argprefix):] + if name not in available: + available.append([name, factory]) + if available: + pluginname = plugin.__name__ + for name, factory in available: + loc = self.getlocation(factory) + if verbose: + funcargspec = "%s -- %s" %(name, loc,) + else: + funcargspec = name + tw.line(funcargspec, green=True) + doc = factory.__doc__ or "" + if doc: + for line in doc.split("\n"): + tw.line(" " + line.strip()) + else: + tw.line(" %s: no docstring available" %(loc,), + red=True) + + def getlocation(self, function): + import inspect + fn = py.path.local(inspect.getfile(function)) + lineno = py.builtin._getcode(function).co_firstlineno + if fn.relto(self.fspath): + fn = fn.relto(self.fspath) + return "%s:%d" %(fn, lineno+1) Added: pypy/branch/py12/py/_plugin/pytest_tmpdir.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_tmpdir.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,22 @@ +"""provide temporary directories to test functions. + +usage example:: + + def test_plugin(tmpdir): + tmpdir.join("hello").write("hello") + +.. _`py.path.local`: ../../path.html + +""" +import py + +def pytest_funcarg__tmpdir(request): + """return a temporary directory path object + unique to each test function invocation, + created as a sub directory of the base temporary + directory. The returned object is a `py.path.local`_ + path object. + """ + name = request.function.__name__ + x = request.config.mktemp(name, numbered=True) + return x.realpath() Added: pypy/branch/py12/py/_plugin/pytest_unittest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/pytest_unittest.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,81 @@ +""" +automatically discover and run traditional "unittest.py" style tests. + +Usage +---------------- + +This plugin collects and runs Python `unittest.py style`_ tests. +It will automatically collect ``unittest.TestCase`` subclasses +and their ``test`` methods from the test modules of a project +(usually following the ``test_*.py`` pattern). + +This plugin is enabled by default. + +.. _`unittest.py style`: http://docs.python.org/library/unittest.html +""" +import py +import sys + +def pytest_pycollect_makeitem(collector, name, obj): + if 'unittest' not in sys.modules: + return # nobody derived unittest.TestCase + try: + isunit = issubclass(obj, py.std.unittest.TestCase) + except KeyboardInterrupt: + raise + except Exception: + pass + else: + if isunit: + return UnitTestCase(name, parent=collector) + +class UnitTestCase(py.test.collect.Class): + def collect(self): + return [UnitTestCaseInstance("()", self)] + + def setup(self): + pass + + def teardown(self): + pass + +_dummy = object() +class UnitTestCaseInstance(py.test.collect.Instance): + def collect(self): + loader = py.std.unittest.TestLoader() + names = loader.getTestCaseNames(self.obj.__class__) + l = [] + for name in names: + callobj = getattr(self.obj, name) + if py.builtin.callable(callobj): + l.append(UnitTestFunction(name, parent=self)) + return l + + def _getobj(self): + x = self.parent.obj + return self.parent.obj(methodName='run') + +class UnitTestFunction(py.test.collect.Function): + def __init__(self, name, parent, args=(), obj=_dummy, sort_value=None): + super(UnitTestFunction, self).__init__(name, parent) + self._args = args + if obj is not _dummy: + self._obj = obj + self._sort_value = sort_value + if hasattr(self.parent, 'newinstance'): + self.parent.newinstance() + self.obj = self._getobj() + + def runtest(self): + target = self.obj + args = self._args + target(*args) + + def setup(self): + instance = py.builtin._getimself(self.obj) + instance.setUp() + + def teardown(self): + instance = py.builtin._getimself(self.obj) + instance.tearDown() + Added: pypy/branch/py12/py/_plugin/standalonetemplate.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_plugin/standalonetemplate.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,63 @@ +#! /usr/bin/env python + +sources = """ + at SOURCES@""" + +import sys +import base64 +import zlib +import imp + +class DictImporter(object): + def __init__(self, sources): + self.sources = sources + + def find_module(self, fullname, path=None): + if fullname in self.sources: + return self + if fullname+'.__init__' in self.sources: + return self + return None + + def load_module(self, fullname): + # print "load_module:", fullname + from types import ModuleType + try: + s = self.sources[fullname] + is_pkg = False + except KeyError: + s = self.sources[fullname+'.__init__'] + is_pkg = True + + co = compile(s, fullname, 'exec') + module = sys.modules.setdefault(fullname, ModuleType(fullname)) + module.__file__ = "%s/%s" % (__file__, fullname) + module.__loader__ = self + if is_pkg: + module.__path__ = [fullname] + + do_exec(co, module.__dict__) + return sys.modules[fullname] + + def get_source(self, name): + res = self.sources.get(name) + if res is None: + res = self.sources.get(name+'.__init__') + return res + +if __name__ == "__main__": + if sys.version_info >= (3,0): + exec("def do_exec(co, loc): exec(co, loc)\n") + import pickle + sources = sources.encode("ascii") # ensure bytes + sources = pickle.loads(zlib.decompress(base64.decodebytes(sources))) + else: + import cPickle as pickle + exec("def do_exec(co, loc): exec co in loc\n") + sources = pickle.loads(zlib.decompress(base64.decodestring(sources))) + + importer = DictImporter(sources) + sys.meta_path.append(importer) + + import py + py.cmdline.pytest() Added: pypy/branch/py12/py/_process/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_process/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +""" high-level sub-process handling """ Added: pypy/branch/py12/py/_process/cmdexec.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_process/cmdexec.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,46 @@ +""" + +""" + +import os, sys +import subprocess +import py +from subprocess import Popen, PIPE + +def cmdexec(cmd): + """ return output of executing 'cmd' in a separate process. + + raise cmdexec.ExecutionFailed exeception if the command failed. + the exception will provide an 'err' attribute containing + the error-output from the command. + """ + process = subprocess.Popen(cmd, shell=True, + universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + out = py.builtin._totext(out, sys.getdefaultencoding()) + err = py.builtin._totext(err, sys.getdefaultencoding()) + status = process.poll() + if status: + raise ExecutionFailed(status, status, cmd, out, err) + return out + +class ExecutionFailed(py.error.Error): + def __init__(self, status, systemstatus, cmd, out, err): + Exception.__init__(self) + self.status = status + self.systemstatus = systemstatus + self.cmd = cmd + self.err = err + self.out = out + + def __str__(self): + return "ExecutionFailed: %d %s\n%s" %(self.status, self.cmd, self.err) + +# export the exception under the name 'py.process.cmdexec.Error' +cmdexec.Error = ExecutionFailed +try: + ExecutionFailed.__module__ = 'py.process.cmdexec' + ExecutionFailed.__name__ = 'Error' +except (AttributeError, TypeError): + pass Added: pypy/branch/py12/py/_process/forkedfunc.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_process/forkedfunc.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,108 @@ + +""" + ForkedFunc provides a way to run a function in a forked process + and get at its return value, stdout and stderr output as well + as signals and exitstatusus. + + XXX see if tempdir handling is sane +""" + +import py +import os +import sys +import marshal + +class ForkedFunc(object): + EXITSTATUS_EXCEPTION = 3 + def __init__(self, fun, args=None, kwargs=None, nice_level=0): + if args is None: + args = [] + if kwargs is None: + kwargs = {} + self.fun = fun + self.args = args + self.kwargs = kwargs + self.tempdir = tempdir = py.path.local.mkdtemp() + self.RETVAL = tempdir.ensure('retval') + self.STDOUT = tempdir.ensure('stdout') + self.STDERR = tempdir.ensure('stderr') + + pid = os.fork() + if pid: # in parent process + self.pid = pid + else: # in child process + self._child(nice_level) + + def _child(self, nice_level): + # right now we need to call a function, but first we need to + # map all IO that might happen + # make sure sys.stdout points to file descriptor one + sys.stdout = stdout = self.STDOUT.open('w') + sys.stdout.flush() + fdstdout = stdout.fileno() + if fdstdout != 1: + os.dup2(fdstdout, 1) + sys.stderr = stderr = self.STDERR.open('w') + fdstderr = stderr.fileno() + if fdstderr != 2: + os.dup2(fdstderr, 2) + retvalf = self.RETVAL.open("wb") + EXITSTATUS = 0 + try: + if nice_level: + os.nice(nice_level) + try: + retval = self.fun(*self.args, **self.kwargs) + retvalf.write(marshal.dumps(retval)) + except: + excinfo = py.code.ExceptionInfo() + stderr.write(excinfo.exconly()) + EXITSTATUS = self.EXITSTATUS_EXCEPTION + finally: + stdout.close() + stderr.close() + retvalf.close() + os.close(1) + os.close(2) + os._exit(EXITSTATUS) + + def waitfinish(self, waiter=os.waitpid): + pid, systemstatus = waiter(self.pid, 0) + if systemstatus: + if os.WIFSIGNALED(systemstatus): + exitstatus = os.WTERMSIG(systemstatus) + 128 + else: + exitstatus = os.WEXITSTATUS(systemstatus) + #raise ExecutionFailed(status, systemstatus, cmd, + # ''.join(out), ''.join(err)) + else: + exitstatus = 0 + signal = systemstatus & 0x7f + if not exitstatus and not signal: + retval = self.RETVAL.open('rb') + try: + retval_data = retval.read() + finally: + retval.close() + retval = marshal.loads(retval_data) + else: + retval = None + stdout = self.STDOUT.read() + stderr = self.STDERR.read() + self._removetemp() + return Result(exitstatus, signal, retval, stdout, stderr) + + def _removetemp(self): + if self.tempdir.check(): + self.tempdir.remove() + + def __del__(self): + self._removetemp() + +class Result(object): + def __init__(self, exitstatus, signal, retval, stdout, stderr): + self.exitstatus = exitstatus + self.signal = signal + self.retval = retval + self.out = stdout + self.err = stderr Added: pypy/branch/py12/py/_process/killproc.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_process/killproc.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,23 @@ +import py +import os, sys + +if sys.platform == "win32" or getattr(os, '_name', '') == 'nt': + try: + import ctypes + except ImportError: + def dokill(pid): + py.process.cmdexec("taskkill /F /PID %d" %(pid,)) + else: + def dokill(pid): + PROCESS_TERMINATE = 1 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_TERMINATE, False, pid) + ctypes.windll.kernel32.TerminateProcess(handle, -1) + ctypes.windll.kernel32.CloseHandle(handle) +else: + def dokill(pid): + os.kill(pid, 15) + +def kill(pid): + """ kill process by id. """ + dokill(pid) Added: pypy/branch/py12/py/_std.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_std.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,18 @@ +import sys + +class Std(object): + """ makes top-level python modules available as an attribute, + importing them on first access. + """ + + def __init__(self): + self.__dict__ = sys.modules + + def __getattr__(self, name): + try: + m = __import__(name) + except ImportError: + raise AttributeError("py.std: could not import %s" % name) + return m + +std = Std() Added: pypy/branch/py12/py/_test/.pluginmanager.py.swp ============================================================================== Binary file. No diff available. Added: pypy/branch/py12/py/_test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/__init__.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1 @@ +""" assertion and py.test helper API.""" Added: pypy/branch/py12/py/_test/cmdline.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/cmdline.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,24 @@ +import py +import sys + +# +# main entry point +# + +def main(args=None): + if args is None: + args = sys.argv[1:] + config = py.test.config + try: + config.parse(args) + config.pluginmanager.do_configure(config) + session = config.initsession() + colitems = config.getinitialnodes() + exitstatus = session.main(colitems) + config.pluginmanager.do_unconfigure(config) + raise SystemExit(exitstatus) + except config.Error: + e = sys.exc_info()[1] + sys.stderr.write("ERROR: %s\n" %(e.args[0],)) + raise SystemExit(3) + Added: pypy/branch/py12/py/_test/collect.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/collect.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,417 @@ +""" +test collection nodes, forming a tree, Items are leafs. +""" +import py + +def configproperty(name): + def fget(self): + #print "retrieving %r property from %s" %(name, self.fspath) + return self.config._getcollectclass(name, self.fspath) + return property(fget) + +class HookProxy: + def __init__(self, node): + self.node = node + def __getattr__(self, name): + if name[0] == "_": + raise AttributeError(name) + hookmethod = getattr(self.node.config.hook, name) + def call_matching_hooks(**kwargs): + plugins = self.node.config._getmatchingplugins(self.node.fspath) + return hookmethod.pcall(plugins, **kwargs) + return call_matching_hooks + +class Node(object): + """ base class for all Nodes in the collection tree. + Collector subclasses have children, Items are terminal nodes. + """ + def __init__(self, name, parent=None, config=None): + self.name = name + self.parent = parent + self.config = config or parent.config + self.fspath = getattr(parent, 'fspath', None) + self.ihook = HookProxy(self) + + def _reraiseunpicklingproblem(self): + if hasattr(self, '_unpickle_exc'): + py.builtin._reraise(*self._unpickle_exc) + + # + # note to myself: Pickling is uh. + # + def __getstate__(self): + return (self.name, self.parent) + def __setstate__(self, nameparent): + name, parent = nameparent + try: + colitems = parent._memocollect() + for colitem in colitems: + if colitem.name == name: + # we are a copy that will not be returned + # by our parent + self.__dict__ = colitem.__dict__ + break + else: + raise ValueError("item %r not found in parent collection %r" %( + name, [x.name for x in colitems])) + except KeyboardInterrupt: + raise + except Exception: + # our parent can't collect us but we want unpickling to + # otherwise continue - self._reraiseunpicklingproblem() will + # reraise the problem + self._unpickle_exc = py.std.sys.exc_info() + self.name = name + self.parent = parent + self.config = parent.config + + def __repr__(self): + if getattr(self.config.option, 'debug', False): + return "<%s %r %0x>" %(self.__class__.__name__, + getattr(self, 'name', None), id(self)) + else: + return "<%s %r>" %(self.__class__.__name__, + getattr(self, 'name', None)) + + # methods for ordering nodes + + def __eq__(self, other): + if not isinstance(other, Node): + return False + return self.name == other.name and self.parent == other.parent + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.name, self.parent)) + + def setup(self): + pass + + def teardown(self): + pass + + def _memoizedcall(self, attrname, function): + exattrname = "_ex_" + attrname + failure = getattr(self, exattrname, None) + if failure is not None: + py.builtin._reraise(failure[0], failure[1], failure[2]) + if hasattr(self, attrname): + return getattr(self, attrname) + try: + res = function() + except (KeyboardInterrupt, SystemExit): + raise + except: + failure = py.std.sys.exc_info() + setattr(self, exattrname, failure) + raise + setattr(self, attrname, res) + return res + + def listchain(self): + """ return list of all parent collectors up to self, + starting from root of collection tree. """ + l = [self] + while 1: + x = l[0] + if x.parent is not None and x.parent.parent is not None: + l.insert(0, x.parent) + else: + return l + + def listnames(self): + return [x.name for x in self.listchain()] + + def getparent(self, cls): + current = self + while current and not isinstance(current, cls): + current = current.parent + return current + + def readkeywords(self): + return dict([(x, True) for x in self._keywords()]) + + def _keywords(self): + return [self.name] + + def _skipbykeyword(self, keywordexpr): + """ return True if they given keyword expression means to + skip this collector/item. + """ + if not keywordexpr: + return + chain = self.listchain() + for key in filter(None, keywordexpr.split()): + eor = key[:1] == '-' + if eor: + key = key[1:] + if not (eor ^ self._matchonekeyword(key, chain)): + return True + + def _matchonekeyword(self, key, chain): + elems = key.split(".") + # XXX O(n^2), anyone cares? + chain = [item.readkeywords() for item in chain if item._keywords()] + for start, _ in enumerate(chain): + if start + len(elems) > len(chain): + return False + for num, elem in enumerate(elems): + for keyword in chain[num + start]: + ok = False + if elem in keyword: + ok = True + break + if not ok: + break + if num == len(elems) - 1 and ok: + return True + return False + + def _prunetraceback(self, traceback): + return traceback + + def _repr_failure_py(self, excinfo): + excinfo.traceback = self._prunetraceback(excinfo.traceback) + # XXX should excinfo.getrepr record all data and toterminal() + # process it? + if self.config.option.tbstyle == "short": + style = "short" + else: + style = "long" + return excinfo.getrepr(funcargs=True, + showlocals=self.config.option.showlocals, + style=style) + + repr_failure = _repr_failure_py + shortfailurerepr = "F" + +class Collector(Node): + """ + Collector instances create children through collect() + and thus iteratively build a tree. attributes:: + + parent: attribute pointing to the parent collector + (or None if this is the root collector) + name: basename of this collector object + """ + Directory = configproperty('Directory') + Module = configproperty('Module') + + def collect(self): + """ returns a list of children (items and collectors) + for this collection node. + """ + raise NotImplementedError("abstract") + + def collect_by_name(self, name): + """ return a child matching the given name, else None. """ + for colitem in self._memocollect(): + if colitem.name == name: + return colitem + + def repr_failure(self, excinfo, outerr=None): + """ represent a failure. """ + assert outerr is None, "XXX deprecated" + return self._repr_failure_py(excinfo) + + def _memocollect(self): + """ internal helper method to cache results of calling collect(). """ + return self._memoizedcall('_collected', self.collect) + + # ********************************************************************** + # DEPRECATED METHODS + # ********************************************************************** + + def _deprecated_collect(self): + # avoid recursion: + # collect -> _deprecated_collect -> custom run() -> + # super().run() -> collect + attrname = '_depcollectentered' + if hasattr(self, attrname): + return + setattr(self, attrname, True) + method = getattr(self.__class__, 'run', None) + if method is not None and method != Collector.run: + warnoldcollect(function=method) + names = self.run() + return [x for x in [self.join(name) for name in names] if x] + + def run(self): + """ DEPRECATED: returns a list of names available from this collector. + You can return an empty list. Callers of this method + must take care to catch exceptions properly. + """ + return [colitem.name for colitem in self._memocollect()] + + def join(self, name): + """ DEPRECATED: return a child collector or item for the given name. + If the return value is None there is no such child. + """ + return self.collect_by_name(name) + + def _prunetraceback(self, traceback): + if hasattr(self, 'fspath'): + path = self.fspath + ntraceback = traceback.cut(path=self.fspath) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._pydir) + traceback = ntraceback.filter() + return traceback + +class FSCollector(Collector): + def __init__(self, fspath, parent=None, config=None): + fspath = py.path.local(fspath) + super(FSCollector, self).__init__(fspath.basename, parent, config=config) + self.fspath = fspath + + def __getstate__(self): + # RootCollector.getbynames() inserts a directory which we need + # to throw out here for proper re-instantiation + if isinstance(self.parent.parent, RootCollector): + assert self.parent.fspath == self.parent.parent.fspath, self.parent + return (self.name, self.parent.parent) # shortcut + return super(Collector, self).__getstate__() + +class File(FSCollector): + """ base class for collecting tests from a file. """ + +class Directory(FSCollector): + def recfilter(self, path): + if path.check(dir=1, dotfile=0): + return path.basename not in ('CVS', '_darcs', '{arch}') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + l = [] + for path in self.fspath.listdir(sort=True): + res = self.consider(path) + if res is not None: + if isinstance(res, (list, tuple)): + l.extend(res) + else: + l.append(res) + return l + + def consider(self, path): + if self.ihook.pytest_ignore_collect_path(path=path, config=self.config): + return + if path.check(file=1): + res = self.consider_file(path) + elif path.check(dir=1): + res = self.consider_dir(path) + else: + res = None + if isinstance(res, list): + # throw out identical results + l = [] + for x in res: + if x not in l: + assert x.parent == self, (x.parent, self) + assert x.fspath == path, (x.fspath, path) + l.append(x) + res = l + return res + + def consider_file(self, path): + return self.ihook.pytest_collect_file(path=path, parent=self) + + def consider_dir(self, path, usefilters=None): + if usefilters is not None: + py.log._apiwarn("0.99", "usefilters argument not needed") + return self.ihook.pytest_collect_directory(path=path, parent=self) + +class Item(Node): + """ a basic test item. """ + def _deprecated_testexecution(self): + if self.__class__.run != Item.run: + warnoldtestrun(function=self.run) + elif self.__class__.execute != Item.execute: + warnoldtestrun(function=self.execute) + else: + return False + self.run() + return True + + def run(self): + """ deprecated, here because subclasses might call it. """ + return self.execute(self.obj) + + def execute(self, obj): + """ deprecated, here because subclasses might call it. """ + return obj() + + def reportinfo(self): + return self.fspath, None, "" + +def warnoldcollect(function=None): + py.log._apiwarn("1.0", + "implement collector.collect() instead of " + "collector.run() and collector.join()", + stacklevel=2, function=function) + +def warnoldtestrun(function=None): + py.log._apiwarn("1.0", + "implement item.runtest() instead of " + "item.run() and item.execute()", + stacklevel=2, function=function) + + + +class RootCollector(Directory): + def __init__(self, config): + Directory.__init__(self, config.topdir, parent=None, config=config) + self.name = None + + def __repr__(self): + return "" %(self.fspath,) + + def getbynames(self, names): + current = self.consider(self.config.topdir) + while names: + name = names.pop(0) + if name == ".": # special "identity" name + continue + l = [] + for x in current._memocollect(): + if x.name == name: + l.append(x) + elif x.fspath == current.fspath.join(name): + l.append(x) + elif x.name == "()": + names.insert(0, name) + l.append(x) + break + if not l: + raise ValueError("no node named %r below %r" %(name, current)) + current = l[0] + return current + + def totrail(self, node): + chain = node.listchain() + names = [self._getrelpath(chain[0].fspath)] + names += [x.name for x in chain[1:]] + return names + + def fromtrail(self, trail): + return self.config._rootcol.getbynames(trail) + + def _getrelpath(self, fspath): + topdir = self.config.topdir + relpath = fspath.relto(topdir) + if not relpath: + if fspath == topdir: + relpath = "." + else: + raise ValueError("%r not relative to topdir %s" + %(self.fspath, topdir)) + return relpath + + def __getstate__(self): + return self.config + + def __setstate__(self, config): + self.__init__(config) Added: pypy/branch/py12/py/_test/config.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/config.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,291 @@ +import py, os +from py._test.conftesthandle import Conftest +from py._test.pluginmanager import PluginManager +from py._test import parseopt +from py._test.collect import RootCollector + +def ensuretemp(string, dir=1): + """ (deprecated) return temporary directory path with + the given string as the trailing part. It is usually + better to use the 'tmpdir' function argument which will + take care to provide empty unique directories for each + test call even if the test is called multiple times. + """ + #py.log._apiwarn(">1.1", "use tmpdir function argument") + return py.test.config.ensuretemp(string, dir=dir) + +class CmdOptions(object): + """ holds cmdline options as attributes.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + def __repr__(self): + return "" %(self.__dict__,) + +class Error(Exception): + """ Test Configuration Error. """ + +class Config(object): + """ access to config values, pluginmanager and plugin hooks. """ + Option = py.std.optparse.Option + Error = Error + basetemp = None + _sessionclass = None + + def __init__(self, topdir=None, option=None): + self.option = option or CmdOptions() + self.topdir = topdir + self._parser = parseopt.Parser( + usage="usage: %prog [options] [file_or_dir] [file_or_dir] [...]", + processopt=self._processopt, + ) + self.pluginmanager = PluginManager() + self._conftest = Conftest(onimport=self._onimportconftest) + self.hook = self.pluginmanager.hook + + def _onimportconftest(self, conftestmodule): + self.trace("loaded conftestmodule %r" %(conftestmodule,)) + self.pluginmanager.consider_conftest(conftestmodule) + + def _getmatchingplugins(self, fspath): + allconftests = self._conftest._conftestpath2mod.values() + plugins = [x for x in self.pluginmanager.getplugins() + if x not in allconftests] + plugins += self._conftest.getconftestmodules(fspath) + return plugins + + def trace(self, msg): + if getattr(self.option, 'traceconfig', None): + self.hook.pytest_trace(category="config", msg=msg) + + def _processopt(self, opt): + if hasattr(opt, 'default') and opt.dest: + val = os.environ.get("PYTEST_OPTION_" + opt.dest.upper(), None) + if val is not None: + if opt.type == "int": + val = int(val) + elif opt.type == "long": + val = long(val) + elif opt.type == "float": + val = float(val) + elif not opt.type and opt.action in ("store_true", "store_false"): + val = eval(val) + opt.default = val + else: + name = "option_" + opt.dest + try: + opt.default = self._conftest.rget(name) + except (ValueError, KeyError): + pass + if not hasattr(self.option, opt.dest): + setattr(self.option, opt.dest, opt.default) + + def _preparse(self, args): + self.pluginmanager.consider_setuptools_entrypoints() + self.pluginmanager.consider_env() + self.pluginmanager.consider_preparse(args) + self._conftest.setinitial(args) + self.pluginmanager.do_addoption(self._parser) + + def parse(self, args): + """ parse cmdline arguments into this config object. + Note that this can only be called once per testing process. + """ + assert not hasattr(self, 'args'), ( + "can only parse cmdline args at most once per Config object") + self._preparse(args) + self._parser.hints.extend(self.pluginmanager._hints) + args = self._parser.parse_setoption(args, self.option) + if not args: + args.append(py.std.os.getcwd()) + self.topdir = gettopdir(args) + self._rootcol = RootCollector(config=self) + self._setargs(args) + + def _setargs(self, args): + self.args = list(args) + self._argfspaths = [py.path.local(decodearg(x)[0]) for x in args] + + # config objects are usually pickled across system + # barriers but they contain filesystem paths. + # upon getstate/setstate we take care to do everything + # relative to "topdir". + def __getstate__(self): + l = [] + for path in self.args: + path = py.path.local(path) + l.append(path.relto(self.topdir)) + return l, self.option.__dict__ + + def __setstate__(self, repr): + # we have to set py.test.config because loading + # of conftest files may use it (deprecated) + # mainly by py.test.config.addoptions() + global config_per_process + py.test.config = config_per_process = self + args, cmdlineopts = repr + cmdlineopts = CmdOptions(**cmdlineopts) + # next line will registers default plugins + self.__init__(topdir=py.path.local(), option=cmdlineopts) + self._rootcol = RootCollector(config=self) + args = [str(self.topdir.join(x)) for x in args] + self._preparse(args) + self._setargs(args) + + def ensuretemp(self, string, dir=True): + return self.getbasetemp().ensure(string, dir=dir) + + def getbasetemp(self): + if self.basetemp is None: + basetemp = self.option.basetemp + if basetemp: + basetemp = py.path.local(basetemp) + if not basetemp.check(dir=1): + basetemp.mkdir() + else: + basetemp = py.path.local.make_numbered_dir(prefix='pytest-') + self.basetemp = basetemp + return self.basetemp + + def mktemp(self, basename, numbered=False): + basetemp = self.getbasetemp() + if not numbered: + return basetemp.mkdir(basename) + else: + return py.path.local.make_numbered_dir(prefix=basename, + keep=0, rootdir=basetemp, lock_timeout=None) + + def getinitialnodes(self): + return [self.getnode(arg) for arg in self.args] + + def getnode(self, arg): + parts = decodearg(arg) + path = py.path.local(parts.pop(0)) + if not path.check(): + raise self.Error("file not found: %s" %(path,)) + topdir = self.topdir + if path != topdir and not path.relto(topdir): + raise self.Error("path %r is not relative to %r" % + (str(path), str(topdir))) + # assumtion: pytest's fs-collector tree follows the filesystem tree + names = list(filter(None, path.relto(topdir).split(path.sep))) + names += parts + try: + return self._rootcol.getbynames(names) + except ValueError: + e = py.std.sys.exc_info()[1] + raise self.Error("can't collect: %s\n%s" % (arg, e.args[0])) + + def _getcollectclass(self, name, path): + try: + cls = self._conftest.rget(name, path) + except KeyError: + return getattr(py.test.collect, name) + else: + py.log._apiwarn(">1.1", "%r was found in a conftest.py file, " + "use pytest_collect hooks instead." % (cls,)) + return cls + + def getconftest_pathlist(self, name, path=None): + """ return a matching value, which needs to be sequence + of filenames that will be returned as a list of Path + objects (they can be relative to the location + where they were found). + """ + try: + mod, relroots = self._conftest.rget_with_confmod(name, path) + except KeyError: + return None + modpath = py.path.local(mod.__file__).dirpath() + l = [] + for relroot in relroots: + if not isinstance(relroot, py.path.local): + relroot = relroot.replace("/", py.path.local.sep) + relroot = modpath.join(relroot, abs=True) + l.append(relroot) + return l + + def addoptions(self, groupname, *specs): + """ add a named group of options to the current testing session. + This function gets invoked during testing session initialization. + """ + py.log._apiwarn("1.0", "define pytest_addoptions(parser) to add options", stacklevel=2) + group = self._parser.getgroup(groupname) + for opt in specs: + group._addoption_instance(opt) + return self.option + + def addoption(self, *optnames, **attrs): + return self._parser.addoption(*optnames, **attrs) + + def getvalueorskip(self, name, path=None): + """ return getvalue() or call py.test.skip if no value exists. """ + try: + val = self.getvalue(name, path) + if val is None: + raise KeyError(name) + return val + except KeyError: + py.test.skip("no %r value found" %(name,)) + + def getvalue(self, name, path=None): + """ return 'name' value looked up from the 'options' + and then from the first conftest file found up + the path (including the path itself). + if path is None, lookup the value in the initial + conftest modules found during command line parsing. + """ + try: + return getattr(self.option, name) + except AttributeError: + return self._conftest.rget(name, path) + + def setsessionclass(self, cls): + if self._sessionclass is not None: + raise ValueError("sessionclass already set to: %r" %( + self._sessionclass)) + self._sessionclass = cls + + def initsession(self): + """ return an initialized session object. """ + cls = self._sessionclass + if cls is None: + from py._test.session import Session + cls = Session + session = cls(self) + self.trace("instantiated session %r" % session) + return session + +# +# helpers +# + +def gettopdir(args): + """ return the top directory for the given paths. + if the common base dir resides in a python package + parent directory of the root package is returned. + """ + fsargs = [py.path.local(decodearg(arg)[0]) for arg in args] + p = fsargs and fsargs[0] or None + for x in fsargs[1:]: + p = p.common(x) + assert p, "cannot determine common basedir of %s" %(fsargs,) + pkgdir = p.pypkgpath() + if pkgdir is None: + if p.check(file=1): + p = p.dirpath() + return p + else: + return pkgdir.dirpath() + +def decodearg(arg): + arg = str(arg) + return arg.split("::") + +def onpytestaccess(): + # it's enough to have our containing module loaded as + # it initializes a per-process config instance + # which loads default plugins which add to py.test.* + pass + +# a default per-process instance of py.test configuration +config_per_process = Config() Added: pypy/branch/py12/py/_test/conftesthandle.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/conftesthandle.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,113 @@ +import py + +class Conftest(object): + """ the single place for accessing values and interacting + towards conftest modules from py.test objects. + + (deprecated) + Note that triggering Conftest instances to import + conftest.py files may result in added cmdline options. + """ + def __init__(self, onimport=None, confcutdir=None): + self._path2confmods = {} + self._onimport = onimport + self._conftestpath2mod = {} + self._confcutdir = confcutdir + + def setinitial(self, args): + """ try to find a first anchor path for looking up global values + from conftests. This function is usually called _before_ + argument parsing. conftest files may add command line options + and we thus have no completely safe way of determining + which parts of the arguments are actually related to options + and which are file system paths. We just try here to get + bootstrapped ... + """ + current = py.path.local() + opt = '--confcutdir' + for i in range(len(args)): + opt1 = str(args[i]) + if opt1.startswith(opt): + if opt1 == opt: + if len(args) > i: + p = current.join(args[i+1], abs=True) + elif opt1.startswith(opt + "="): + p = current.join(opt1[len(opt)+1:], abs=1) + self._confcutdir = p + break + for arg in args + [current]: + anchor = current.join(arg, abs=1) + if anchor.check(): # we found some file object + self._path2confmods[None] = self.getconftestmodules(anchor) + # let's also consider test* dirs + if anchor.check(dir=1): + for x in anchor.listdir(lambda x: x.check(dir=1, dotfile=0)): + self.getconftestmodules(x) + break + else: + assert 0, "no root of filesystem?" + + def getconftestmodules(self, path): + """ return a list of imported conftest modules for the given path. """ + try: + clist = self._path2confmods[path] + except KeyError: + if path is None: + raise ValueError("missing default confest.") + dp = path.dirpath() + if dp == path: + clist = [] + else: + cutdir = self._confcutdir + clist = self.getconftestmodules(dp) + if cutdir and path != cutdir and not path.relto(cutdir): + pass + else: + conftestpath = path.join("conftest.py") + if conftestpath.check(file=1): + clist.append(self.importconftest(conftestpath)) + self._path2confmods[path] = clist + # be defensive: avoid changes from caller side to + # affect us by always returning a copy of the actual list + return clist[:] + + def rget(self, name, path=None): + mod, value = self.rget_with_confmod(name, path) + return value + + def rget_with_confmod(self, name, path=None): + modules = self.getconftestmodules(path) + modules.reverse() + for mod in modules: + try: + return mod, getattr(mod, name) + except AttributeError: + continue + raise KeyError(name) + + def importconftest(self, conftestpath): + assert conftestpath.check(), conftestpath + try: + return self._conftestpath2mod[conftestpath] + except KeyError: + if not conftestpath.dirpath('__init__.py').check(file=1): + # HACK: we don't want any "globally" imported conftest.py, + # prone to conflicts and subtle problems + modname = str(conftestpath).replace('.', conftestpath.sep) + mod = conftestpath.pyimport(modname=modname) + else: + mod = conftestpath.pyimport() + self._conftestpath2mod[conftestpath] = mod + dirpath = conftestpath.dirpath() + if dirpath in self._path2confmods: + for path, mods in self._path2confmods.items(): + if path and path.relto(dirpath) or path == dirpath: + assert mod not in mods + mods.append(mod) + self._postimport(mod) + return mod + + def _postimport(self, mod): + if self._onimport: + self._onimport(mod) + return mod Added: pypy/branch/py12/py/_test/funcargs.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/funcargs.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,176 @@ +import py + +def getfuncargnames(function): + argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0] + startindex = py.std.inspect.ismethod(function) and 1 or 0 + defaults = getattr(function, 'func_defaults', + getattr(function, '__defaults__', None)) or () + numdefaults = len(defaults) + if numdefaults: + return argnames[startindex:-numdefaults] + return argnames[startindex:] + +def fillfuncargs(function): + """ fill missing funcargs. """ + request = FuncargRequest(pyfuncitem=function) + request._fillfuncargs() + +def getplugins(node, withpy=False): # might by any node + plugins = node.config._getmatchingplugins(node.fspath) + if withpy: + mod = node.getparent(py.test.collect.Module) + if mod is not None: + plugins.append(mod.obj) + inst = node.getparent(py.test.collect.Instance) + if inst is not None: + plugins.append(inst.obj) + return plugins + +_notexists = object() +class CallSpec: + def __init__(self, funcargs, id, param): + self.funcargs = funcargs + self.id = id + if param is not _notexists: + self.param = param + def __repr__(self): + return "" %( + self.id, getattr(self, 'param', '?'), self.funcargs) + +class Metafunc: + def __init__(self, function, config=None, cls=None, module=None): + self.config = config + self.module = module + self.function = function + self.funcargnames = getfuncargnames(function) + self.cls = cls + self.module = module + self._calls = [] + self._ids = py.builtin.set() + + def addcall(self, funcargs=None, id=_notexists, param=_notexists): + assert funcargs is None or isinstance(funcargs, dict) + if id is None: + raise ValueError("id=None not allowed") + if id is _notexists: + id = len(self._calls) + id = str(id) + if id in self._ids: + raise ValueError("duplicate id %r" % id) + self._ids.add(id) + self._calls.append(CallSpec(funcargs, id, param)) + +class FuncargRequest: + _argprefix = "pytest_funcarg__" + _argname = None + + class LookupError(LookupError): + """ error on performing funcarg request. """ + + def __init__(self, pyfuncitem): + self._pyfuncitem = pyfuncitem + self.function = pyfuncitem.obj + self.module = pyfuncitem.getparent(py.test.collect.Module).obj + clscol = pyfuncitem.getparent(py.test.collect.Class) + self.cls = clscol and clscol.obj or None + self.instance = py.builtin._getimself(self.function) + self.config = pyfuncitem.config + self.fspath = pyfuncitem.fspath + if hasattr(pyfuncitem, '_requestparam'): + self.param = pyfuncitem._requestparam + self._plugins = getplugins(pyfuncitem, withpy=True) + self._funcargs = self._pyfuncitem.funcargs.copy() + self._name2factory = {} + self._currentarg = None + + def _fillfuncargs(self): + argnames = getfuncargnames(self.function) + if argnames: + assert not getattr(self._pyfuncitem, '_args', None), ( + "yielded functions cannot have funcargs") + for argname in argnames: + if argname not in self._pyfuncitem.funcargs: + self._pyfuncitem.funcargs[argname] = self.getfuncargvalue(argname) + + def cached_setup(self, setup, teardown=None, scope="module", extrakey=None): + """ cache and return result of calling setup(). + + The requested argument name, the scope and the ``extrakey`` + determine the cache key. The scope also determines when + teardown(result) will be called. valid scopes are: + scope == 'function': when the single test function run finishes. + scope == 'module': when tests in a different module are run + scope == 'session': when tests of the session have run. + """ + if not hasattr(self.config, '_setupcache'): + self.config._setupcache = {} # XXX weakref? + cachekey = (self._currentarg, self._getscopeitem(scope), extrakey) + cache = self.config._setupcache + try: + val = cache[cachekey] + except KeyError: + val = setup() + cache[cachekey] = val + if teardown is not None: + def finalizer(): + del cache[cachekey] + teardown(val) + self._addfinalizer(finalizer, scope=scope) + return val + + def getfuncargvalue(self, argname): + try: + return self._funcargs[argname] + except KeyError: + pass + if argname not in self._name2factory: + self._name2factory[argname] = self.config.pluginmanager.listattr( + plugins=self._plugins, + attrname=self._argprefix + str(argname) + ) + #else: we are called recursively + if not self._name2factory[argname]: + self._raiselookupfailed(argname) + funcargfactory = self._name2factory[argname].pop() + oldarg = self._currentarg + self._currentarg = argname + try: + self._funcargs[argname] = res = funcargfactory(request=self) + finally: + self._currentarg = oldarg + return res + + def _getscopeitem(self, scope): + if scope == "function": + return self._pyfuncitem + elif scope == "module": + return self._pyfuncitem.getparent(py.test.collect.Module) + elif scope == "session": + return None + raise ValueError("unknown finalization scope %r" %(scope,)) + + def _addfinalizer(self, finalizer, scope): + colitem = self._getscopeitem(scope) + self.config._setupstate.addfinalizer( + finalizer=finalizer, colitem=colitem) + + def addfinalizer(self, finalizer): + """ call the given finalizer after test function finished execution. """ + self._addfinalizer(finalizer, scope="function") + + def __repr__(self): + return "" %(self._pyfuncitem) + + def _raiselookupfailed(self, argname): + available = [] + for plugin in self._plugins: + for name in vars(plugin): + if name.startswith(self._argprefix): + name = name[len(self._argprefix):] + if name not in available: + available.append(name) + fspath, lineno, msg = self._pyfuncitem.reportinfo() + msg = "LookupError: no factory found for function argument %r" % (argname,) + msg += "\n available funcargs: %s" %(", ".join(available),) + msg += "\n use 'py.test --funcargs [testpath]' for help on them." + raise self.LookupError(msg) Added: pypy/branch/py12/py/_test/parseopt.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/parseopt.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,112 @@ +""" +thin wrapper around Python's optparse.py +adding some extra checks and ways to systematically +have Environment variables provide default values +for options. basic usage: + + >>> parser = Parser() + >>> parser.addoption("--hello", action="store_true", dest="hello") + >>> option, args = parser.parse(['--hello']) + >>> option.hello + True + >>> args + [] + +""" +import py +import optparse + +class Parser: + """ Parser for command line arguments. """ + + def __init__(self, usage=None, processopt=None): + self._anonymous = OptionGroup("custom options", parser=self) + self._groups = [] + self._processopt = processopt + self._usage = usage + self.hints = [] + + def processoption(self, option): + if self._processopt: + if option.dest: + self._processopt(option) + + def addnote(self, note): + self._notes.append(note) + + def getgroup(self, name, description="", after=None): + for group in self._groups: + if group.name == name: + return group + group = OptionGroup(name, description, parser=self) + i = 0 + for i, grp in enumerate(self._groups): + if grp.name == after: + break + self._groups.insert(i+1, group) + return group + + addgroup = getgroup + def addgroup(self, name, description=""): + py.log._apiwarn("1.1", "use getgroup() which gets-or-creates") + return self.getgroup(name, description) + + def addoption(self, *opts, **attrs): + """ add an optparse-style option. """ + self._anonymous.addoption(*opts, **attrs) + + def parse(self, args): + optparser = MyOptionParser(self) + groups = self._groups + [self._anonymous] + for group in groups: + if group.options: + desc = group.description or group.name + optgroup = optparse.OptionGroup(optparser, desc) + optgroup.add_options(group.options) + optparser.add_option_group(optgroup) + return optparser.parse_args([str(x) for x in args]) + + def parse_setoption(self, args, option): + parsedoption, args = self.parse(args) + for name, value in parsedoption.__dict__.items(): + setattr(option, name, value) + return args + + +class OptionGroup: + def __init__(self, name, description="", parser=None): + self.name = name + self.description = description + self.options = [] + self.parser = parser + + def addoption(self, *optnames, **attrs): + """ add an option to this group. """ + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=False) + + def _addoption(self, *optnames, **attrs): + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=True) + + def _addoption_instance(self, option, shortupper=False): + if not shortupper: + for opt in option._short_opts: + if opt[0] == '-' and opt[1].islower(): + raise ValueError("lowercase shortoptions reserved") + if self.parser: + self.parser.processoption(option) + self.options.append(option) + + +class MyOptionParser(optparse.OptionParser): + def __init__(self, parser): + self._parser = parser + optparse.OptionParser.__init__(self, usage=parser._usage) + def format_epilog(self, formatter): + hints = self._parser.hints + if hints: + s = "\n".join(["hint: " + x for x in hints]) + "\n" + s = "\n" + s + "\n" + return s + return "" Added: pypy/branch/py12/py/_test/pluginmanager.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/pluginmanager.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,345 @@ +""" +managing loading and interacting with pytest plugins. +""" +import py +import inspect +from py._plugin import hookspec + +default_plugins = ( + "default runner capture mark terminal skipping tmpdir monkeypatch " + "recwarn pdb pastebin unittest helpconfig nose assertion genscript " + "junitxml doctest").split() + +def check_old_use(mod, modname): + clsname = modname[len('pytest_'):].capitalize() + "Plugin" + assert not hasattr(mod, clsname), (mod, clsname) + +class PluginManager(object): + def __init__(self): + self.registry = Registry() + self._name2plugin = {} + self._hints = [] + self.hook = HookRelay([hookspec], registry=self.registry) + self.register(self) + for spec in default_plugins: + self.import_plugin(spec) + + def _getpluginname(self, plugin, name): + if name is None: + if hasattr(plugin, '__name__'): + name = plugin.__name__.split(".")[-1] + else: + name = id(plugin) + return name + + def register(self, plugin, name=None): + assert not self.isregistered(plugin), plugin + assert not self.registry.isregistered(plugin), plugin + name = self._getpluginname(plugin, name) + if name in self._name2plugin: + return False + self._name2plugin[name] = plugin + self.call_plugin(plugin, "pytest_registerhooks", + {'pluginmanager': self}) + self.hook.pytest_plugin_registered(manager=self, plugin=plugin) + self.registry.register(plugin) + return True + + def unregister(self, plugin): + self.hook.pytest_plugin_unregistered(plugin=plugin) + self.registry.unregister(plugin) + for name, value in list(self._name2plugin.items()): + if value == plugin: + del self._name2plugin[name] + + def isregistered(self, plugin, name=None): + if self._getpluginname(plugin, name) in self._name2plugin: + return True + for val in self._name2plugin.values(): + if plugin == val: + return True + + def registerhooks(self, spec): + self.hook._registerhooks(spec) + + def getplugins(self): + return list(self.registry) + + def skipifmissing(self, name): + if not self.hasplugin(name): + py.test.skip("plugin %r is missing" % name) + + def hasplugin(self, name): + try: + self.getplugin(name) + except KeyError: + return False + else: + return True + + def getplugin(self, name): + try: + return self._name2plugin[name] + except KeyError: + impname = canonical_importname(name) + return self._name2plugin[impname] + + # API for bootstrapping + # + def _envlist(self, varname): + val = py.std.os.environ.get(varname, None) + if val is not None: + return val.split(',') + return () + + def consider_env(self): + for spec in self._envlist("PYTEST_PLUGINS"): + self.import_plugin(spec) + + def consider_setuptools_entrypoints(self): + try: + from pkg_resources import iter_entry_points + except ImportError: + return # XXX issue a warning + for ep in iter_entry_points('pytest11'): + name = canonical_importname(ep.name) + if name in self._name2plugin: + continue + plugin = ep.load() + self.register(plugin, name=name) + + def consider_preparse(self, args): + for opt1,opt2 in zip(args, args[1:]): + if opt1 == "-p": + self.import_plugin(opt2) + + def consider_conftest(self, conftestmodule): + cls = getattr(conftestmodule, 'ConftestPlugin', None) + if cls is not None: + raise ValueError("%r: 'ConftestPlugins' only existed till 1.0.0b1, " + "were removed in 1.0.0b2" % (cls,)) + if self.register(conftestmodule, name=conftestmodule.__file__): + self.consider_module(conftestmodule) + + def consider_module(self, mod): + attr = getattr(mod, "pytest_plugins", ()) + if attr: + if not isinstance(attr, (list, tuple)): + attr = (attr,) + for spec in attr: + self.import_plugin(spec) + + def import_plugin(self, spec): + assert isinstance(spec, str) + modname = canonical_importname(spec) + if modname in self._name2plugin: + return + try: + mod = importplugin(modname) + except KeyboardInterrupt: + raise + except py.test.skip.Exception: + e = py.std.sys.exc_info()[1] + self._hints.append("skipped plugin %r: %s" %((modname, e.msg))) + else: + check_old_use(mod, modname) + self.register(mod) + self.consider_module(mod) + + def pytest_terminal_summary(self, terminalreporter): + tw = terminalreporter._tw + if terminalreporter.config.option.traceconfig: + for hint in self._hints: + tw.line("hint: %s" % hint) + + # + # + # API for interacting with registered and instantiated plugin objects + # + # + def listattr(self, attrname, plugins=None): + return self.registry.listattr(attrname, plugins=plugins) + + def notify_exception(self, excinfo=None): + if excinfo is None: + excinfo = py.code.ExceptionInfo() + excrepr = excinfo.getrepr(funcargs=True, showlocals=True) + return self.hook.pytest_internalerror(excrepr=excrepr) + + def do_addoption(self, parser): + mname = "pytest_addoption" + methods = self.registry.listattr(mname, reverse=True) + mc = MultiCall(methods, {'parser': parser}) + mc.execute() + + def pytest_plugin_registered(self, plugin): + dic = self.call_plugin(plugin, "pytest_namespace", {}) or {} + for name, value in dic.items(): + setattr(py.test, name, value) + py.test.__all__.append(name) + if hasattr(self, '_config'): + self.call_plugin(plugin, "pytest_addoption", + {'parser': self._config._parser}) + self.call_plugin(plugin, "pytest_configure", + {'config': self._config}) + + def call_plugin(self, plugin, methname, kwargs): + return MultiCall( + methods=self.listattr(methname, plugins=[plugin]), + kwargs=kwargs, firstresult=True).execute() + + def do_configure(self, config): + assert not hasattr(self, '_config') + self._config = config + config.hook.pytest_configure(config=self._config) + + def do_unconfigure(self, config): + config = self._config + del self._config + config.hook.pytest_unconfigure(config=config) + config.pluginmanager.unregister(self) + +def canonical_importname(name): + name = name.lower() + modprefix = "pytest_" + if not name.startswith(modprefix): + name = modprefix + name + return name + +def importplugin(importspec): + try: + return __import__(importspec) + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + try: + return __import__("py._plugin.%s" %(importspec), + None, None, '__doc__') + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + # show the original exception, not the failing internal one + return __import__(importspec) + + +class MultiCall: + """ execute a call into multiple python functions/methods. """ + + def __init__(self, methods, kwargs, firstresult=False): + self.methods = methods[:] + self.kwargs = kwargs.copy() + self.kwargs['__multicall__'] = self + self.results = [] + self.firstresult = firstresult + + def __repr__(self): + status = "%d results, %d meths" % (len(self.results), len(self.methods)) + return "" %(status, self.kwargs) + + def execute(self): + while self.methods: + method = self.methods.pop() + kwargs = self.getkwargs(method) + res = method(**kwargs) + if res is not None: + self.results.append(res) + if self.firstresult: + return res + if not self.firstresult: + return self.results + + def getkwargs(self, method): + kwargs = {} + for argname in varnames(method): + try: + kwargs[argname] = self.kwargs[argname] + except KeyError: + pass # might be optional param + return kwargs + +def varnames(func): + ismethod = inspect.ismethod(func) + rawcode = py.code.getrawcode(func) + try: + return rawcode.co_varnames[ismethod:] + except AttributeError: + return () + +class Registry: + """ + Manage Plugins: register/unregister call calls to plugins. + """ + def __init__(self, plugins=None): + if plugins is None: + plugins = [] + self._plugins = plugins + + def register(self, plugin): + assert not isinstance(plugin, str) + assert not plugin in self._plugins + self._plugins.append(plugin) + + def unregister(self, plugin): + self._plugins.remove(plugin) + + def isregistered(self, plugin): + return plugin in self._plugins + + def __iter__(self): + return iter(self._plugins) + + def listattr(self, attrname, plugins=None, reverse=False): + l = [] + if plugins is None: + plugins = self._plugins + for plugin in plugins: + try: + l.append(getattr(plugin, attrname)) + except AttributeError: + continue + if reverse: + l.reverse() + return l + +class HookRelay: + def __init__(self, hookspecs, registry): + if not isinstance(hookspecs, list): + hookspecs = [hookspecs] + self._hookspecs = [] + self._registry = registry + for hookspec in hookspecs: + self._registerhooks(hookspec) + + def _registerhooks(self, hookspecs): + self._hookspecs.append(hookspecs) + for name, method in vars(hookspecs).items(): + if name[:1] != "_": + firstresult = getattr(method, 'firstresult', False) + hc = HookCaller(self, name, firstresult=firstresult) + setattr(self, name, hc) + #print ("setting new hook", name) + + def _performcall(self, name, multicall): + return multicall.execute() + +class HookCaller: + def __init__(self, hookrelay, name, firstresult): + self.hookrelay = hookrelay + self.name = name + self.firstresult = firstresult + + def __repr__(self): + return "" %(self.name,) + + def __call__(self, **kwargs): + methods = self.hookrelay._registry.listattr(self.name) + mc = MultiCall(methods, kwargs, firstresult=self.firstresult) + return self.hookrelay._performcall(self.name, mc) + + def pcall(self, plugins, **kwargs): + methods = self.hookrelay._registry.listattr(self.name, plugins=plugins) + mc = MultiCall(methods, kwargs, firstresult=self.firstresult) + return self.hookrelay._performcall(self.name, mc) + Added: pypy/branch/py12/py/_test/pycollect.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/pycollect.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,397 @@ +""" +Python related collection nodes. +""" +import py +import inspect +from py._test.collect import configproperty, warnoldcollect +from py._test import funcargs +from py._code.code import TerminalRepr + +class PyobjMixin(object): + def obj(): + def fget(self): + try: + return self._obj + except AttributeError: + self._obj = obj = self._getobj() + return obj + def fset(self, value): + self._obj = value + return property(fget, fset, None, "underlying python object") + obj = obj() + + def _getobj(self): + return getattr(self.parent.obj, self.name) + + def getmodpath(self, stopatmodule=True, includemodule=False): + """ return python path relative to the containing module. """ + chain = self.listchain() + chain.reverse() + parts = [] + for node in chain: + if isinstance(node, Instance): + continue + name = node.name + if isinstance(node, Module): + assert name.endswith(".py") + name = name[:-3] + if stopatmodule: + if includemodule: + parts.append(name) + break + parts.append(name) + parts.reverse() + s = ".".join(parts) + return s.replace(".[", "[") + + def _getfslineno(self): + try: + return self._fslineno + except AttributeError: + pass + obj = self.obj + # xxx let decorators etc specify a sane ordering + if hasattr(obj, 'place_as'): + obj = obj.place_as + + self._fslineno = py.code.getfslineno(obj) + return self._fslineno + + def reportinfo(self): + fspath, lineno = self._getfslineno() + modpath = self.getmodpath() + return fspath, lineno, modpath + +class PyCollectorMixin(PyobjMixin, py.test.collect.Collector): + Class = configproperty('Class') + Instance = configproperty('Instance') + Function = configproperty('Function') + Generator = configproperty('Generator') + + def funcnamefilter(self, name): + return name.startswith('test') + def classnamefilter(self, name): + return name.startswith('Test') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + # NB. we avoid random getattrs and peek in the __dict__ instead + dicts = [getattr(self.obj, '__dict__', {})] + for basecls in inspect.getmro(self.obj.__class__): + dicts.append(basecls.__dict__) + seen = {} + l = [] + for dic in dicts: + for name, obj in dic.items(): + if name in seen: + continue + seen[name] = True + if name[0] != "_": + res = self.makeitem(name, obj) + if res is None: + continue + if not isinstance(res, list): + res = [res] + l.extend(res) + l.sort(key=lambda item: item.reportinfo()[:2]) + return l + + def _deprecated_join(self, name): + if self.__class__.join != py.test.collect.Collector.join: + warnoldcollect() + return self.join(name) + + def makeitem(self, name, obj): + return self.ihook.pytest_pycollect_makeitem( + collector=self, name=name, obj=obj) + + def _istestclasscandidate(self, name, obj): + if self.classnamefilter(name) and \ + inspect.isclass(obj): + if hasinit(obj): + # XXX WARN + return False + return True + + def _genfunctions(self, name, funcobj): + module = self.getparent(Module).obj + clscol = self.getparent(Class) + cls = clscol and clscol.obj or None + metafunc = funcargs.Metafunc(funcobj, config=self.config, + cls=cls, module=module) + gentesthook = self.config.hook.pytest_generate_tests + plugins = funcargs.getplugins(self, withpy=True) + gentesthook.pcall(plugins, metafunc=metafunc) + if not metafunc._calls: + return self.Function(name, parent=self) + l = [] + for callspec in metafunc._calls: + subname = "%s[%s]" %(name, callspec.id) + function = self.Function(name=subname, parent=self, + callspec=callspec, callobj=funcobj) + l.append(function) + return l + +class Module(py.test.collect.File, PyCollectorMixin): + def _getobj(self): + return self._memoizedcall('_obj', self._importtestmodule) + + def _importtestmodule(self): + # we assume we are only called once per module + mod = self.fspath.pyimport() + #print "imported test module", mod + self.config.pluginmanager.consider_module(mod) + return mod + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.log._apiwarn(">1.1.1", "%r uses 'disabled' which is deprecated, " + "use pytestmark=..., see pytest_skipping plugin" % (self.obj,)) + py.test.skip("%r is disabled" %(self.obj,)) + if hasattr(self.obj, 'setup_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.setup_module)[0]: + self.obj.setup_module(self.obj) + else: + self.obj.setup_module() + + def teardown(self): + if hasattr(self.obj, 'teardown_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.teardown_module)[0]: + self.obj.teardown_module(self.obj) + else: + self.obj.teardown_module() + +class Class(PyCollectorMixin, py.test.collect.Collector): + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + return [self.Instance(name="()", parent=self)] + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.log._apiwarn(">1.1.1", "%r uses 'disabled' which is deprecated, " + "use pytestmark=..., see pytest_skipping plugin" % (self.obj,)) + py.test.skip("%r is disabled" %(self.obj,)) + setup_class = getattr(self.obj, 'setup_class', None) + if setup_class is not None: + setup_class = getattr(setup_class, 'im_func', setup_class) + setup_class(self.obj) + + def teardown(self): + teardown_class = getattr(self.obj, 'teardown_class', None) + if teardown_class is not None: + teardown_class = getattr(teardown_class, 'im_func', teardown_class) + teardown_class(self.obj) + +class Instance(PyCollectorMixin, py.test.collect.Collector): + def _getobj(self): + return self.parent.obj() + def Function(self): + return getattr(self.obj, 'Function', + PyCollectorMixin.Function.__get__(self)) # XXX for python 2.2 + def _keywords(self): + return [] + Function = property(Function) + + #def __repr__(self): + # return "<%s of '%s'>" %(self.__class__.__name__, + # self.parent.obj.__name__) + + def newinstance(self): + self.obj = self._getobj() + return self.obj + +class FunctionMixin(PyobjMixin): + """ mixin for the code common to Function and Generator. + """ + + def setup(self): + """ perform setup for this test function. """ + if inspect.ismethod(self.obj): + name = 'setup_method' + else: + name = 'setup_function' + if isinstance(self.parent, Instance): + obj = self.parent.newinstance() + self.obj = self._getobj() + else: + obj = self.parent.obj + setup_func_or_method = getattr(obj, name, None) + if setup_func_or_method is not None: + setup_func_or_method(self.obj) + + def teardown(self): + """ perform teardown for this test function. """ + if inspect.ismethod(self.obj): + name = 'teardown_method' + else: + name = 'teardown_function' + obj = self.parent.obj + teardown_func_or_meth = getattr(obj, name, None) + if teardown_func_or_meth is not None: + teardown_func_or_meth(self.obj) + + def _prunetraceback(self, traceback): + if hasattr(self, '_obj') and not self.config.option.fulltrace: + code = py.code.Code(self.obj) + path, firstlineno = code.path, code.firstlineno + ntraceback = traceback.cut(path=path, firstlineno=firstlineno) + if ntraceback == traceback: + ntraceback = ntraceback.cut(path=path) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._pydir) + traceback = ntraceback.filter() + return traceback + + def _repr_failure_py(self, excinfo): + if excinfo.errisinstance(funcargs.FuncargRequest.LookupError): + fspath, lineno, msg = self.reportinfo() + lines, _ = inspect.getsourcelines(self.obj) + for i, line in enumerate(lines): + if line.strip().startswith('def'): + return FuncargLookupErrorRepr(fspath, lineno, + lines[:i+1], str(excinfo.value)) + return super(FunctionMixin, self)._repr_failure_py(excinfo) + + def repr_failure(self, excinfo, outerr=None): + assert outerr is None, "XXX outerr usage is deprecated" + return self._repr_failure_py(excinfo) + + shortfailurerepr = "F" + +class FuncargLookupErrorRepr(TerminalRepr): + def __init__(self, filename, firstlineno, deflines, errorstring): + self.deflines = deflines + self.errorstring = errorstring + self.filename = filename + self.firstlineno = firstlineno + + def toterminal(self, tw): + tw.line() + for line in self.deflines: + tw.line(" " + line.strip()) + for line in self.errorstring.split("\n"): + tw.line(" " + line.strip(), red=True) + tw.line() + tw.line("%s:%d" % (self.filename, self.firstlineno+1)) + +class Generator(FunctionMixin, PyCollectorMixin, py.test.collect.Collector): + def collect(self): + # test generators are seen as collectors but they also + # invoke setup/teardown on popular request + # (induced by the common "test_*" naming shared with normal tests) + self.config._setupstate.prepare(self) + l = [] + seen = {} + for i, x in enumerate(self.obj()): + name, call, args = self.getcallargs(x) + if not py.builtin.callable(call): + raise TypeError("%r yielded non callable test %r" %(self.obj, call,)) + if name is None: + name = "[%d]" % i + else: + name = "['%s']" % name + if name in seen: + raise ValueError("%r generated tests with non-unique name %r" %(self, name)) + seen[name] = True + l.append(self.Function(name, self, args=args, callobj=call)) + return l + + def getcallargs(self, obj): + if not isinstance(obj, (tuple, list)): + obj = (obj,) + # explict naming + if isinstance(obj[0], py.builtin._basestring): + name = obj[0] + obj = obj[1:] + else: + name = None + call, args = obj[0], obj[1:] + return name, call, args + + +# +# Test Items +# +_dummy = object() +class Function(FunctionMixin, py.test.collect.Item): + """ a Function Item is responsible for setting up + and executing a Python callable test object. + """ + _genid = None + def __init__(self, name, parent=None, args=None, config=None, + callspec=None, callobj=_dummy): + super(Function, self).__init__(name, parent, config=config) + self._args = args + if self._isyieldedfunction(): + assert not callspec, "yielded functions (deprecated) cannot have funcargs" + else: + if callspec is not None: + self.funcargs = callspec.funcargs or {} + self._genid = callspec.id + if hasattr(callspec, "param"): + self._requestparam = callspec.param + else: + self.funcargs = {} + if callobj is not _dummy: + self._obj = callobj + self.function = getattr(self.obj, 'im_func', self.obj) + + def _getobj(self): + name = self.name + i = name.find("[") # parametrization + if i != -1: + name = name[:i] + return getattr(self.parent.obj, name) + + def _isyieldedfunction(self): + return self._args is not None + + def readkeywords(self): + d = super(Function, self).readkeywords() + d.update(py.builtin._getfuncdict(self.obj)) + return d + + def runtest(self): + """ execute the underlying test function. """ + self.ihook.pytest_pyfunc_call(pyfuncitem=self) + + def setup(self): + super(Function, self).setup() + if hasattr(self, 'funcargs'): + funcargs.fillfuncargs(self) + + def __eq__(self, other): + try: + return (self.name == other.name and + self._args == other._args and + self.parent == other.parent and + self.obj == other.obj and + getattr(self, '_genid', None) == + getattr(other, '_genid', None) + ) + except AttributeError: + pass + return False + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.parent, self.name)) + +def hasinit(obj): + init = getattr(obj, '__init__', None) + if init: + if not isinstance(init, type(object.__init__)): + return True Added: pypy/branch/py12/py/_test/session.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_test/session.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,128 @@ +""" basic test session implementation. + +* drives collection of tests +* triggers executions of tests +* produces events used by reporting +""" + +import py + +# exitcodes for the command line +EXIT_OK = 0 +EXIT_TESTSFAILED = 1 +EXIT_INTERRUPTED = 2 +EXIT_INTERNALERROR = 3 +EXIT_NOHOSTS = 4 + +# imports used for genitems() +Item = py.test.collect.Item +Collector = py.test.collect.Collector + +class Session(object): + nodeid = "" + def __init__(self, config): + self.config = config + self.pluginmanager = config.pluginmanager # shortcut + self.pluginmanager.register(self) + self._testsfailed = False + self._nomatch = False + self.shouldstop = False + + def genitems(self, colitems, keywordexpr=None): + """ yield Items from iterating over the given colitems. """ + if colitems: + colitems = list(colitems) + while colitems: + next = colitems.pop(0) + if isinstance(next, (tuple, list)): + colitems[:] = list(next) + colitems + continue + assert self.pluginmanager is next.config.pluginmanager + if isinstance(next, Item): + remaining = self.filteritems([next]) + if remaining: + self.config.hook.pytest_itemstart(item=next) + yield next + else: + assert isinstance(next, Collector) + self.config.hook.pytest_collectstart(collector=next) + rep = self.config.hook.pytest_make_collect_report(collector=next) + if rep.passed: + for x in self.genitems(rep.result, keywordexpr): + yield x + self.config.hook.pytest_collectreport(report=rep) + if self.shouldstop: + break + + def filteritems(self, colitems): + """ return items to process (some may be deselected)""" + keywordexpr = self.config.option.keyword + if not keywordexpr or self._nomatch: + return colitems + if keywordexpr[-1] == ":": + keywordexpr = keywordexpr[:-1] + remaining = [] + deselected = [] + for colitem in colitems: + if isinstance(colitem, Item): + if colitem._skipbykeyword(keywordexpr): + deselected.append(colitem) + continue + remaining.append(colitem) + if deselected: + self.config.hook.pytest_deselected(items=deselected) + if self.config.option.keyword.endswith(":"): + self._nomatch = True + return remaining + + def collect(self, colitems): + keyword = self.config.option.keyword + for x in self.genitems(colitems, keyword): + yield x + + def sessionstarts(self): + """ setup any neccessary resources ahead of the test run. """ + self.config.hook.pytest_sessionstart(session=self) + + def pytest_runtest_logreport(self, report): + if report.failed: + self._testsfailed = True + if self.config.option.exitfirst: + self.shouldstop = True + pytest_collectreport = pytest_runtest_logreport + + def sessionfinishes(self, exitstatus): + """ teardown any resources after a test run. """ + self.config.hook.pytest_sessionfinish( + session=self, + exitstatus=exitstatus, + ) + + def main(self, colitems): + """ main loop for running tests. """ + self.shouldstop = False + self.sessionstarts() + exitstatus = EXIT_OK + try: + self._mainloop(colitems) + if self._testsfailed: + exitstatus = EXIT_TESTSFAILED + self.sessionfinishes(exitstatus=exitstatus) + except KeyboardInterrupt: + excinfo = py.code.ExceptionInfo() + self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + exitstatus = EXIT_INTERRUPTED + except: + excinfo = py.code.ExceptionInfo() + self.config.pluginmanager.notify_exception(excinfo) + exitstatus = EXIT_INTERNALERROR + if exitstatus in (EXIT_INTERNALERROR, EXIT_INTERRUPTED): + self.sessionfinishes(exitstatus=exitstatus) + return exitstatus + + def _mainloop(self, colitems): + for item in self.collect(colitems): + if self.shouldstop: + break + if not self.config.option.collectonly: + item.config.hook.pytest_runtest_protocol(item=item) Added: pypy/branch/py12/py/_xmlgen.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/_xmlgen.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,244 @@ +""" +module for generating and serializing xml and html structures +by using simple python objects. + +(c) holger krekel, holger at merlinux eu. 2009 +""" +import py +import sys, re + +if sys.version_info >= (3,0): + def u(s): + return s + def unicode(x): + if hasattr(x, '__unicode__'): + return x.__unicode__() + return str(x) +else: + def u(s): + return unicode(s) + unicode = unicode + + +class NamespaceMetaclass(type): + def __getattr__(self, name): + if name[:1] == '_': + raise AttributeError(name) + if self == Namespace: + raise ValueError("Namespace class is abstract") + tagspec = self.__tagspec__ + if tagspec is not None and name not in tagspec: + raise AttributeError(name) + classattr = {} + if self.__stickyname__: + classattr['xmlname'] = name + cls = type(name, (self.__tagclass__,), classattr) + setattr(self, name, cls) + return cls + +class Tag(list): + class Attr(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __init__(self, *args, **kwargs): + super(Tag, self).__init__(args) + self.attr = self.Attr(**kwargs) + + def __unicode__(self): + return self.unicode(indent=0) + __str__ = __unicode__ + + def unicode(self, indent=2): + l = [] + SimpleUnicodeVisitor(l.append, indent).visit(self) + return "".join(l) + + def __repr__(self): + name = self.__class__.__name__ + return "<%r tag object %d>" % (name, id(self)) + +Namespace = NamespaceMetaclass('Namespace', (object, ), { + '__tagspec__': None, + '__tagclass__': Tag, + '__stickyname__': False, +}) + +class HtmlTag(Tag): + def unicode(self, indent=2): + l = [] + HtmlVisitor(l.append, indent, shortempty=False).visit(self) + return u("").join(l) + +# exported plain html namespace +class html(Namespace): + __tagclass__ = HtmlTag + __stickyname__ = True + __tagspec__ = dict([(x,1) for x in ( + 'a,abbr,acronym,address,applet,area,b,bdo,big,blink,' + 'blockquote,body,br,button,caption,center,cite,code,col,' + 'colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed,' + 'fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html,' + 'i,iframe,img,input,ins,kbd,label,legend,li,link,listing,' + 'map,marquee,menu,meta,multicol,nobr,noembed,noframes,' + 'noscript,object,ol,optgroup,option,p,pre,q,s,script,' + 'select,small,span,strike,strong,style,sub,sup,table,' + 'tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp,' + 'base,basefont,frame,hr,isindex,param,samp,var' + ).split(',') if x]) + + class Style(object): + def __init__(self, **kw): + for x, y in kw.items(): + x = x.replace('_', '-') + setattr(self, x, y) + + +class raw(object): + """just a box that can contain a unicode string that will be + included directly in the output""" + def __init__(self, uniobj): + self.uniobj = uniobj + +class SimpleUnicodeVisitor(object): + """ recursive visitor to write unicode. """ + def __init__(self, write, indent=0, curindent=0, shortempty=True): + self.write = write + self.cache = {} + self.visited = {} # for detection of recursion + self.indent = indent + self.curindent = curindent + self.parents = [] + self.shortempty = shortempty # short empty tags or not + + def visit(self, node): + """ dispatcher on node's class/bases name. """ + cls = node.__class__ + try: + visitmethod = self.cache[cls] + except KeyError: + for subclass in cls.__mro__: + visitmethod = getattr(self, subclass.__name__, None) + if visitmethod is not None: + break + else: + visitmethod = self.object + self.cache[cls] = visitmethod + visitmethod(node) + + def object(self, obj): + #self.write(obj) + self.write(escape(unicode(obj))) + + def raw(self, obj): + self.write(obj.uniobj) + + def list(self, obj): + assert id(obj) not in self.visited + self.visited[id(obj)] = 1 + map(self.visit, obj) + + def Tag(self, tag): + assert id(tag) not in self.visited + try: + tag.parent = self.parents[-1] + except IndexError: + tag.parent = None + self.visited[id(tag)] = 1 + tagname = getattr(tag, 'xmlname', tag.__class__.__name__) + if self.curindent and not self._isinline(tagname): + self.write("\n" + u(' ') * self.curindent) + if tag: + self.curindent += self.indent + self.write(u('<%s%s>') % (tagname, self.attributes(tag))) + self.parents.append(tag) + for x in tag: + self.visit(x) + self.parents.pop() + self.write(u('') % tagname) + self.curindent -= self.indent + else: + nameattr = tagname+self.attributes(tag) + if self._issingleton(tagname): + self.write(u('<%s/>') % (nameattr,)) + else: + self.write(u('<%s>') % (nameattr, tagname)) + + def attributes(self, tag): + # serialize attributes + attrlist = dir(tag.attr) + attrlist.sort() + l = [] + for name in attrlist: + res = self.repr_attribute(tag.attr, name) + if res is not None: + l.append(res) + l.extend(self.getstyle(tag)) + return u("").join(l) + + def repr_attribute(self, attrs, name): + if name[:2] != '__': + value = getattr(attrs, name) + if name.endswith('_'): + name = name[:-1] + return ' %s="%s"' % (name, escape(unicode(value))) + + def getstyle(self, tag): + """ return attribute list suitable for styling. """ + try: + styledict = tag.style.__dict__ + except AttributeError: + return [] + else: + stylelist = [x+': ' + y for x,y in styledict.items()] + return [u(' style="%s"') % u('; ').join(stylelist)] + + def _issingleton(self, tagname): + """can (and will) be overridden in subclasses""" + return self.shortempty + + def _isinline(self, tagname): + """can (and will) be overridden in subclasses""" + return False + +class HtmlVisitor(SimpleUnicodeVisitor): + + single = dict([(x, 1) for x in + ('br,img,area,param,col,hr,meta,link,base,' + 'input,frame').split(',')]) + inline = dict([(x, 1) for x in + ('a abbr acronym b basefont bdo big br cite code dfn em font ' + 'i img input kbd label q s samp select small span strike ' + 'strong sub sup textarea tt u var'.split(' '))]) + + def repr_attribute(self, attrs, name): + if name == 'class_': + value = getattr(attrs, name) + if value is None: + return + return super(HtmlVisitor, self).repr_attribute(attrs, name) + + def _issingleton(self, tagname): + return tagname in self.single + + def _isinline(self, tagname): + return tagname in self.inline + + +class _escape: + def __init__(self): + self.escape = { + u('"') : u('"'), u('<') : u('<'), u('>') : u('>'), + u('&') : u('&'), u("'") : u('''), + } + self.charef_rex = re.compile(u("|").join(self.escape.keys())) + + def _replacer(self, match): + return self.escape[match.group(0)] + + def __call__(self, ustring): + """ xml-escape the given unicode string. """ + ustring = unicode(ustring) + return self.charef_rex.sub(self._replacer, ustring) + +escape = _escape() Added: pypy/branch/py12/py/apipkg.py ============================================================================== --- (empty file) +++ pypy/branch/py12/py/apipkg.py Fri Apr 30 17:07:52 2010 @@ -0,0 +1,95 @@ +""" +apipkg: control the exported namespace of a python package. + +see http://pypi.python.org/pypi/apipkg + +(c) holger krekel, 2009 - MIT license +""" +import sys +from types import ModuleType + +__version__ = "1.0b6" + +def initpkg(pkgname, exportdefs): + """ initialize given package from the export definitions. """ + mod = ApiModule(pkgname, exportdefs, implprefix=pkgname) + oldmod = sys.modules[pkgname] + mod.__file__ = getattr(oldmod, '__file__', None) + mod.__version__ = getattr(oldmod, '__version__', '0') + for name in ('__path__', '__loader__'): + if hasattr(oldmod, name): + setattr(mod, name, getattr(oldmod, name)) + sys.modules[pkgname] = mod + +def importobj(modpath, attrname): + module = __import__(modpath, None, None, ['__doc__']) + return getattr(module, attrname) + +class ApiModule(ModuleType): + def __init__(self, name, importspec, implprefix=None): + self.__name__ = name + self.__all__ = [x for x in importspec if x != '__onfirstaccess__'] + self.__map__ = {} + self.__implprefix__ = implprefix or name + for name, importspec in importspec.items(): + if isinstance(importspec, dict): + subname = '%s.%s'%(self.__name__, name) + apimod = ApiModule(subname, importspec, implprefix) + sys.modules[subname] = apimod + setattr(self, name, apimod) + else: + modpath, attrname = importspec.split(':') + if modpath[0] == '.': + modpath = implprefix + modpath + if name == '__doc__': + self.__doc__ = importobj(modpath, attrname) + else: + self.__map__[name] = (modpath, attrname) + + def __repr__(self): + l = [] + if hasattr(self, '__version__'): + l.append("version=" + repr(self.__version__)) + if hasattr(self, '__file__'): + l.append('from ' + repr(self.__file__)) + if l: + return '' % (self.__name__, " ".join(l)) + return '' % (self.__name__,) + + def __makeattr(self, name): + """lazily compute value for name or raise AttributeError if unknown.""" + target = None + if '__onfirstaccess__' in self.__map__: + target = self.__map__.pop('__onfirstaccess__') + importobj(*target)() + try: + modpath, attrname = self.__map__[name] + except KeyError: + if target is not None and name != '__onfirstaccess__': + # retry, onfirstaccess might have set attrs + return getattr(self, name) + raise AttributeError(name) + else: + result = importobj(modpath, attrname) + setattr(self, name, result) + try: + del self.__map__[name] + except KeyError: + pass # in a recursive-import situation a double-del can happen + return result + + __getattr__ = __makeattr + + def __dict__(self): + # force all the content of the module to be loaded when __dict__ is read + dictdescr = ModuleType.__dict__['__dict__'] + dict = dictdescr.__get__(self) + if dict is not None: + hasattr(self, 'some') + for name in self.__all__: + try: + self.__makeattr(name) + except AttributeError: + pass + return dict + __dict__ = property(__dict__) From hpk at codespeak.net Fri Apr 30 17:08:39 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 17:08:39 +0200 (CEST) Subject: [pypy-svn] r74290 - pypy/branch/py12/py/bin Message-ID: <20100430150839.1C700282BF2@codespeak.net> Author: hpk Date: Fri Apr 30 17:08:37 2010 New Revision: 74290 Added: pypy/branch/py12/py/bin/ - copied from r74198, pypy/trunk/py/bin/ Log: copying the pypy-special py/bin directory from trunk From hpk at codespeak.net Fri Apr 30 17:11:49 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 17:11:49 +0200 (CEST) Subject: [pypy-svn] r74291 - pypy/branch/py12/py/bin Message-ID: <20100430151149.2BBA7282BF2@codespeak.net> Author: hpk Date: Fri Apr 30 17:11:47 2010 New Revision: 74291 Modified: pypy/branch/py12/py/bin/py.test Log: tweak py/bin/py.test to ignore warnings issued from setuptools that the inlined py lib shadows a properly installed one. Modified: pypy/branch/py12/py/bin/py.test ============================================================================== --- pypy/branch/py12/py/bin/py.test (original) +++ pypy/branch/py12/py/bin/py.test Fri Apr 30 17:11:47 2010 @@ -1,3 +1,10 @@ #!/usr/bin/env python + +# somewhat PYPY specific hack: +# let's make sure setuptools does show a warning when our inlined 'py' +# version shadows a properly installed one. +import warnings +warnings.filterwarnings("ignore", + "Module py was already imported", category=UserWarning) from _findpy import py -py.cmdline.pytest() \ No newline at end of file +py.cmdline.pytest() From arigo at codespeak.net Fri Apr 30 17:14:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 17:14:45 +0200 (CEST) Subject: [pypy-svn] r74292 - pypy/branch/blackhole-improvement/pypy/jit/metainterp Message-ID: <20100430151445.4CD3D282BF2@codespeak.net> Author: arigo Date: Fri Apr 30 17:14:43 2010 New Revision: 74292 Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: Enough to make test_format pass also on pyjitpl. Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/blackhole.py Fri Apr 30 17:14:43 2010 @@ -125,6 +125,8 @@ position += length elif argtype == 'self': value = self + elif argtype == 'cpu': + value = self.cpu elif argtype == 'pc': value = position elif argtype == 'd': @@ -706,67 +708,67 @@ # ---------- # the following operations are directly implemented by the backend - @arguments("self", "i", "d", "R", returns="i") - def bhimpl_residual_call_r_i(self, func, calldescr, args_r): - return self.cpu.bh_call_i(func, calldescr, None, args_r, None) - @arguments("self", "i", "d", "R", returns="r") - def bhimpl_residual_call_r_r(self, func, calldescr, args_r): - return self.cpu.bh_call_r(func, calldescr, None, args_r, None) - @arguments("self", "i", "d", "R", returns="f") - def bhimpl_residual_call_r_f(self, func, calldescr, args_r): - return self.cpu.bh_call_f(func, calldescr, None, args_r, None) - @arguments("self", "i", "d", "R") - def bhimpl_residual_call_r_v(self, func, calldescr, args_r): - self.cpu.bh_call_v(func, calldescr, None, args_r, None) - - @arguments("self", "i", "d", "I", "R", returns="i") - def bhimpl_residual_call_ir_i(self, func, calldescr, args_i, args_r): - return self.cpu.bh_call_i(func, calldescr, args_i, args_r, None) - @arguments("self", "i", "d", "I", "R", returns="r") - def bhimpl_residual_call_ir_r(self, func, calldescr, args_i, args_r): - return self.cpu.bh_call_r(func, calldescr, args_i, args_r, None) - @arguments("self", "i", "d", "I", "R", returns="f") - def bhimpl_residual_call_ir_f(self, func, calldescr, args_i, args_r): - return self.cpu.bh_call_f(func, calldescr, args_i, args_r, None) - @arguments("self", "i", "d", "I", "R") - def bhimpl_residual_call_ir_v(self, func, calldescr, args_i, args_r): - self.cpu.bh_call_v(func, calldescr, args_i, args_r, None) - - @arguments("self", "i", "d", "I", "R", "F", returns="i") - def bhimpl_residual_call_irf_i(self, func, calldescr,args_i,args_r,args_f): - return self.cpu.bh_call_i(func, calldescr, args_i, args_r, args_f) - @arguments("self", "i", "d", "I", "R", "F", returns="r") - def bhimpl_residual_call_irf_r(self, func, calldescr,args_i,args_r,args_f): - return self.cpu.bh_call_r(func, calldescr, args_i, args_r, args_f) - @arguments("self", "i", "d", "I", "R", "F", returns="f") - def bhimpl_residual_call_irf_f(self, func, calldescr,args_i,args_r,args_f): - return self.cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) - @arguments("self", "i", "d", "I", "R", "F") - def bhimpl_residual_call_irf_v(self, func, calldescr,args_i,args_r,args_f): - self.cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) - - @arguments("self", "d", "i", returns="r") - def bhimpl_new_array(self, arraydescr, length): - return self.cpu.bh_new_array(arraydescr, length) - @arguments("self", "d", "r", "i", "r") - def bhimpl_setarrayitem_gc_r(self, arraydescr, array, index, newvalue): - self.cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) - - @arguments("self", "r", "d", returns="i") - def bhimpl_getfield_gc_i(self, struct, fielddescr): - return self.cpu.bh_getfield_gc_i(struct, fielddescr) - @arguments("self", "r", "d", returns="i") - def bhimpl_getfield_gc_c(self, struct, fielddescr): - return self.cpu.bh_getfield_gc_c(struct, fielddescr) - @arguments("self", "r", "d", returns="i") - def bhimpl_getfield_gc_u(self, struct, fielddescr): - return self.cpu.bh_getfield_gc_u(struct, fielddescr) - @arguments("self", "r", "d", returns="r") - def bhimpl_getfield_gc_r(self, struct, fielddescr): - return self.cpu.bh_getfield_gc_r(struct, fielddescr) - @arguments("self", "r", "d", returns="f") - def bhimpl_getfield_gc_f(self, struct, fielddescr): - return self.cpu.bh_getfield_gc_f(struct, fielddescr) + @arguments("cpu", "i", "d", "R", returns="i") + def bhimpl_residual_call_r_i(cpu, func, calldescr, args_r): + return cpu.bh_call_i(func, calldescr, None, args_r, None) + @arguments("cpu", "i", "d", "R", returns="r") + def bhimpl_residual_call_r_r(cpu, func, calldescr, args_r): + return cpu.bh_call_r(func, calldescr, None, args_r, None) + @arguments("cpu", "i", "d", "R", returns="f") + def bhimpl_residual_call_r_f(cpu, func, calldescr, args_r): + return cpu.bh_call_f(func, calldescr, None, args_r, None) + @arguments("cpu", "i", "d", "R") + def bhimpl_residual_call_r_v(cpu, func, calldescr, args_r): + cpu.bh_call_v(func, calldescr, None, args_r, None) + + @arguments("cpu", "i", "d", "I", "R", returns="i") + def bhimpl_residual_call_ir_i(cpu, func, calldescr, args_i, args_r): + return cpu.bh_call_i(func, calldescr, args_i, args_r, None) + @arguments("cpu", "i", "d", "I", "R", returns="r") + def bhimpl_residual_call_ir_r(cpu, func, calldescr, args_i, args_r): + return cpu.bh_call_r(func, calldescr, args_i, args_r, None) + @arguments("cpu", "i", "d", "I", "R", returns="f") + def bhimpl_residual_call_ir_f(cpu, func, calldescr, args_i, args_r): + return cpu.bh_call_f(func, calldescr, args_i, args_r, None) + @arguments("cpu", "i", "d", "I", "R") + def bhimpl_residual_call_ir_v(cpu, func, calldescr, args_i, args_r): + cpu.bh_call_v(func, calldescr, args_i, args_r, None) + + @arguments("cpu", "i", "d", "I", "R", "F", returns="i") + def bhimpl_residual_call_irf_i(cpu, func, calldescr,args_i,args_r,args_f): + return cpu.bh_call_i(func, calldescr, args_i, args_r, args_f) + @arguments("cpu", "i", "d", "I", "R", "F", returns="r") + def bhimpl_residual_call_irf_r(cpu, func, calldescr,args_i,args_r,args_f): + return cpu.bh_call_r(func, calldescr, args_i, args_r, args_f) + @arguments("cpu", "i", "d", "I", "R", "F", returns="f") + def bhimpl_residual_call_irf_f(cpu, func, calldescr,args_i,args_r,args_f): + return cpu.bh_call_f(func, calldescr, args_i, args_r, args_f) + @arguments("cpu", "i", "d", "I", "R", "F") + def bhimpl_residual_call_irf_v(cpu, func, calldescr,args_i,args_r,args_f): + cpu.bh_call_v(func, calldescr, args_i, args_r, args_f) + + @arguments("cpu", "d", "i", returns="r") + def bhimpl_new_array(cpu, arraydescr, length): + return cpu.bh_new_array(arraydescr, length) + @arguments("cpu", "d", "r", "i", "r") + def bhimpl_setarrayitem_gc_r(cpu, arraydescr, array, index, newvalue): + cpu.bh_setarrayitem_gc_r(arraydescr, array, index, newvalue) + + @arguments("cpu", "r", "d", returns="i") + def bhimpl_getfield_gc_i(cpu, struct, fielddescr): + return cpu.bh_getfield_gc_i(struct, fielddescr) + @arguments("cpu", "r", "d", returns="i") + def bhimpl_getfield_gc_c(cpu, struct, fielddescr): + return cpu.bh_getfield_gc_c(struct, fielddescr) + @arguments("cpu", "r", "d", returns="i") + def bhimpl_getfield_gc_u(cpu, struct, fielddescr): + return cpu.bh_getfield_gc_u(struct, fielddescr) + @arguments("cpu", "r", "d", returns="r") + def bhimpl_getfield_gc_r(cpu, struct, fielddescr): + return cpu.bh_getfield_gc_r(struct, fielddescr) + @arguments("cpu", "r", "d", returns="f") + def bhimpl_getfield_gc_f(cpu, struct, fielddescr): + return cpu.bh_getfield_gc_f(struct, fielddescr) bhimpl_getfield_gc_i_pure = bhimpl_getfield_gc_i bhimpl_getfield_gc_c_pure = bhimpl_getfield_gc_c @@ -774,21 +776,21 @@ bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f - @arguments("self", "i", "d", returns="i") - def bhimpl_getfield_raw_i(self, struct, fielddescr): - return self.cpu.bh_getfield_raw_i(struct, fielddescr) - @arguments("self", "i", "d", returns="i") - def bhimpl_getfield_raw_c(self, struct, fielddescr): - return self.cpu.bh_getfield_raw_c(struct, fielddescr) - @arguments("self", "i", "d", returns="i") - def bhimpl_getfield_raw_u(self, struct, fielddescr): - return self.cpu.bh_getfield_raw_u(struct, fielddescr) - @arguments("self", "i", "d", returns="r") - def bhimpl_getfield_raw_r(self, struct, fielddescr): - return self.cpu.bh_getfield_raw_r(struct, fielddescr) - @arguments("self", "i", "d", returns="f") - def bhimpl_getfield_raw_f(self, struct, fielddescr): - return self.cpu.bh_getfield_raw_f(struct, fielddescr) + @arguments("cpu", "i", "d", returns="i") + def bhimpl_getfield_raw_i(cpu, struct, fielddescr): + return cpu.bh_getfield_raw_i(struct, fielddescr) + @arguments("cpu", "i", "d", returns="i") + def bhimpl_getfield_raw_c(cpu, struct, fielddescr): + return cpu.bh_getfield_raw_c(struct, fielddescr) + @arguments("cpu", "i", "d", returns="i") + def bhimpl_getfield_raw_u(cpu, struct, fielddescr): + return cpu.bh_getfield_raw_u(struct, fielddescr) + @arguments("cpu", "i", "d", returns="r") + def bhimpl_getfield_raw_r(cpu, struct, fielddescr): + return cpu.bh_getfield_raw_r(struct, fielddescr) + @arguments("cpu", "i", "d", returns="f") + def bhimpl_getfield_raw_f(cpu, struct, fielddescr): + return cpu.bh_getfield_raw_f(struct, fielddescr) bhimpl_getfield_raw_i_pure = bhimpl_getfield_raw_i bhimpl_getfield_raw_c_pure = bhimpl_getfield_raw_c @@ -796,50 +798,50 @@ bhimpl_getfield_raw_r_pure = bhimpl_getfield_raw_r bhimpl_getfield_raw_f_pure = bhimpl_getfield_raw_f - @arguments("self", "r", "d", "i") - def bhimpl_setfield_gc_i(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) - @arguments("self", "r", "d", "i") - def bhimpl_setfield_gc_c(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) - @arguments("self", "r", "d", "i") - def bhimpl_setfield_gc_u(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) - @arguments("self", "r", "d", "r") - def bhimpl_setfield_gc_r(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) - @arguments("self", "r", "d", "f") - def bhimpl_setfield_gc_f(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) - - @arguments("self", "i", "d", "i") - def bhimpl_setfield_raw_i(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) - @arguments("self", "i", "d", "i") - def bhimpl_setfield_raw_c(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) - @arguments("self", "i", "d", "i") - def bhimpl_setfield_raw_u(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) - @arguments("self", "i", "d", "r") - def bhimpl_setfield_raw_r(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) - @arguments("self", "i", "d", "f") - def bhimpl_setfield_raw_f(self, struct, fielddescr, newvalue): - self.cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) - - @arguments("self", "d", returns="r") - def bhimpl_new(self, descr): - return self.cpu.bh_new(descr) - - @arguments("self", "d", returns="r") - def bhimpl_new_with_vtable(self, descr): - return self.cpu.bh_new_with_vtable(descr) - - @arguments("self", "r", returns="i") - def bhimpl_guard_class(self, struct): - return self.cpu.bh_classof(struct) - - @arguments("self", "r", returns="i") - def bhimpl_cast_ptr_to_int(self, p): - return self.cpu.bh_cast_ptr_to_int(p) + @arguments("cpu", "r", "d", "i") + def bhimpl_setfield_gc_i(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_gc_i(struct, fielddescr, newvalue) + @arguments("cpu", "r", "d", "i") + def bhimpl_setfield_gc_c(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_gc_c(struct, fielddescr, newvalue) + @arguments("cpu", "r", "d", "i") + def bhimpl_setfield_gc_u(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_gc_u(struct, fielddescr, newvalue) + @arguments("cpu", "r", "d", "r") + def bhimpl_setfield_gc_r(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_gc_r(struct, fielddescr, newvalue) + @arguments("cpu", "r", "d", "f") + def bhimpl_setfield_gc_f(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_gc_f(struct, fielddescr, newvalue) + + @arguments("cpu", "i", "d", "i") + def bhimpl_setfield_raw_i(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_raw_i(struct, fielddescr, newvalue) + @arguments("cpu", "i", "d", "i") + def bhimpl_setfield_raw_c(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_raw_c(struct, fielddescr, newvalue) + @arguments("cpu", "i", "d", "i") + def bhimpl_setfield_raw_u(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_raw_u(struct, fielddescr, newvalue) + @arguments("cpu", "i", "d", "r") + def bhimpl_setfield_raw_r(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_raw_r(struct, fielddescr, newvalue) + @arguments("cpu", "i", "d", "f") + def bhimpl_setfield_raw_f(cpu, struct, fielddescr, newvalue): + cpu.bh_setfield_raw_f(struct, fielddescr, newvalue) + + @arguments("cpu", "d", returns="r") + def bhimpl_new(cpu, descr): + return cpu.bh_new(descr) + + @arguments("cpu", "d", returns="r") + def bhimpl_new_with_vtable(cpu, descr): + return cpu.bh_new_with_vtable(descr) + + @arguments("cpu", "r", returns="i") + def bhimpl_guard_class(cpu, struct): + return cpu.bh_classof(struct) + + @arguments("cpu", "r", returns="i") + def bhimpl_cast_ptr_to_int(cpu, p): + return cpu.bh_cast_ptr_to_int(p) Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/executor.py Fri Apr 30 17:14:43 2010 @@ -61,6 +61,21 @@ return None raise AssertionError("bad rettype") +def do_setarrayitem_gc(cpu, arraybox, indexbox, itembox, arraydescr): + array = arraybox.getref_base() + index = indexbox.getint() + if itembox.type == INT: + item = itembox.getint() + cpu.bh_setarrayitem_gc_i(arraydescr, array, index, item) + elif itembox.type == REF: + item = itembox.getref_base() + cpu.bh_setarrayitem_gc_r(arraydescr, array, index, item) + elif itembox.type == FLOAT: + item = itembox.getfloat() + cpu.bh_setarrayitem_gc_f(arraydescr, array, index, item) + else: + raise AssertionError("bad itembox.type") + # ____________________________________________________________ ##def do_force_token(cpu): @@ -114,6 +129,19 @@ raise Exception("duplicate entry for op number %d" % value) if key.endswith('_PURE'): key = key[:-5] + # + # Fish for a way for the pyjitpl interpreter to delegate + # really running the operation to the blackhole interpreter + # or directly to the cpu. First try the do_xxx() functions + # explicitly encoded above: + name = 'do_' + key.lower() + if name in globals(): + execute[value] = globals()[name] + continue + # If missing, fallback to the bhimpl_xxx() method of the + # blackhole interpreter. This only works if there is a + # method of the exact same name and it accepts simple + # parameters. name = 'bhimpl_' + key.lower() if hasattr(BlackholeInterpreter, name): func = make_execute_function_with_boxes( @@ -122,11 +150,7 @@ if func is not None: execute[value] = func continue - name = 'do_' + key.lower() - if name in globals(): - execute[value] = globals()[name] - continue - pass # XXX... + print "XXX warning: missing", key cpuclass._execute_by_num_args = execute_by_num_args def make_execute_function_with_boxes(name, func): @@ -134,7 +158,7 @@ # from the BlackholeInterpreter class. The wrapper is a new function # that receives and returns boxed values. for argtype in func.argtypes: - if argtype not in ('i', 'r', 'f'): + if argtype not in ('i', 'r', 'f', 'd', 'cpu'): return None if func.resulttype not in ('i', 'r', 'f', None): return None @@ -144,11 +168,17 @@ def do(cpu, *argboxes): newargs = () for argtype in argtypes: - argbox = argboxes[0] - argboxes = argboxes[1:] - if argtype == 'i': value = argbox.getint() - elif argtype == 'r': value = argbox.getref_base() - elif argtype == 'f': value = argbox.getfloat() + if argtype == 'cpu': + value = cpu + elif argtype == 'd': + value = argboxes[-1] + argboxes = argboxes[:-1] + else: + argbox = argboxes[0] + argboxes = argboxes[1:] + if argtype == 'i': value = argbox.getint() + elif argtype == 'r': value = argbox.getref_base() + elif argtype == 'f': value = argbox.getfloat() newargs = newargs + (value,) assert not argboxes # @@ -190,7 +220,8 @@ assert descr is None func = get_execute_function(cpu, opnum, len(argboxes)) assert func is not None - return func(cpu, *argboxes) + return func(cpu, *argboxes) # note that the 'argboxes' tuple + # optionally ends with the descr execute._annspecialcase_ = 'specialize:arg(1)' def execute_varargs(cpu, opnum, argboxes, descr): Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri Apr 30 17:14:43 2010 @@ -32,10 +32,11 @@ return func return decorate -def XXX(func): - def cannot_call(*args, **kwds): - raise Exception('not implemented: %s' % func.__name__) - return cannot_call +class XXX: # XXX temporary hack + def __init__(self, func): + self.func = func + def __getattr__(self, _): + raise Exception("@XXX: " + self.func.__name__) # ____________________________________________________________ @@ -279,9 +280,9 @@ def opimpl_subclassof(self, box1, box2): self.execute(rop.SUBCLASSOF, box1, box2) - @XXX #arguments("descr", "box") - def opimpl_new_array(self, itemsize, countbox): - self.execute_with_descr(rop.NEW_ARRAY, itemsize, countbox) + @arguments("descr", "box") + def opimpl_new_array(self, itemsizedescr, countbox): + return self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, countbox) @XXX #arguments("box", "descr", "box") def opimpl_getarrayitem_gc(self, arraybox, arraydesc, indexbox): @@ -291,9 +292,14 @@ def opimpl_getarrayitem_gc_pure(self, arraybox, arraydesc, indexbox): self.execute_with_descr(rop.GETARRAYITEM_GC_PURE, arraydesc, arraybox, indexbox) - @XXX #arguments("box", "descr", "box", "box") - def opimpl_setarrayitem_gc(self, arraybox, arraydesc, indexbox, itembox): - self.execute_with_descr(rop.SETARRAYITEM_GC, arraydesc, arraybox, indexbox, itembox) + @arguments("descr", "box", "box", "box") + def _opimpl_setarrayitem_gc(self, arraydesc, arraybox, indexbox, itembox): + self.execute_with_descr(rop.SETARRAYITEM_GC, arraydesc, arraybox, + indexbox, itembox) + + opimpl_setarrayitem_gc_i = _opimpl_setarrayitem_gc + opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc + opimpl_setarrayitem_gc_f = _opimpl_setarrayitem_gc @XXX #arguments("box", "descr") def opimpl_arraylen_gc(self, arraybox, arraydesc): From arigo at codespeak.net Fri Apr 30 17:38:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Apr 2010 17:38:47 +0200 (CEST) Subject: [pypy-svn] r74293 - in pypy/branch/blackhole-improvement/pypy/jit: codewriter metainterp Message-ID: <20100430153847.AC0C1282BF2@codespeak.net> Author: arigo Date: Fri Apr 30 17:38:46 2010 New Revision: 74293 Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Log: In-progress. Modified: pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/codewriter/assembler.py Fri Apr 30 17:38:46 2010 @@ -1,6 +1,7 @@ from pypy.jit.metainterp.history import AbstractValue, AbstractDescr, getkind from pypy.jit.codewriter.flatten import Register, Label, TLabel, KINDS from pypy.jit.codewriter.flatten import ListOfKind, SwitchDictDescr +from pypy.jit.codewriter.format import format_assembler from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rlib.objectmodel import we_are_translated @@ -100,6 +101,7 @@ self.insns = {} self.descrs = [] self._descr_dict = {} + self._count_jitcodes = 0 def assemble(self, ssarepr): self.setup() @@ -107,7 +109,7 @@ self.write_insn(insn) self.fix_labels() self.check_result() - return self.make_jitcode(ssarepr.name) + return self.make_jitcode(ssarepr) def setup(self): self.code = [] @@ -248,8 +250,8 @@ assert self.count_regs['ref'] + len(self.constants_r) <= 256 assert self.count_regs['float'] + len(self.constants_f) <= 256 - def make_jitcode(self, name): - jitcode = JitCode(name, liveness=self.liveness, + def make_jitcode(self, ssarepr): + jitcode = JitCode(ssarepr.name, liveness=self.liveness, assembler=self) jitcode.setup(''.join(self.code), self.constants_i, @@ -258,4 +260,7 @@ self.count_regs['int'], self.count_regs['ref'], self.count_regs['float']) + if self._count_jitcodes < 50: # stop if we have a lot of them + jitcode._dump = format_assembler(ssarepr) + self._count_jitcodes += 1 return jitcode Modified: pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/blackhole-improvement/pypy/jit/metainterp/pyjitpl.py Fri Apr 30 17:38:46 2010 @@ -183,27 +183,25 @@ def opimpl_goto(self, target): self.pc = target - @XXX #arguments("orgpc", "jumptarget", "box", "varargs") - def opimpl_goto_if_not(self, pc, target, box, livelist): + @arguments("orgpc", "label", "box") + def opimpl_goto_if_not(self, pc, target, box): switchcase = box.getint() if switchcase: opnum = rop.GUARD_TRUE else: self.pc = target opnum = rop.GUARD_FALSE - self.env = livelist self.generate_guard(pc, opnum, box) - # note about handling self.env explicitly here: it is done in - # such a way that the 'box' on which we generate the guard is - # typically not included in the livelist. - - @arguments("label", "box", "box") - def opimpl_goto_if_not_int_lt(self, target, box1, box2): - if box1.getint() < box2.getint(): - pass - else: - self.pc = target + for _opimpl in ['int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', + ]: + exec py.code.Source(''' + @arguments("orgpc", "label", "box", "box") + def opimpl_goto_if_not_%s(self, pc, target, b1, b2): + condbox = self.execute(rop.%s, b1, b2) + self.opimpl_goto_if_not(pc, target, condbox) + ''' % (_opimpl, _opimpl.upper())).compile() + def follow_jump(self): _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not assert ord(self.bytecode[self.pc]) == _op_goto_if_not From afa at codespeak.net Fri Apr 30 18:03:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 18:03:42 +0200 (CEST) Subject: [pypy-svn] r74294 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20100430160342.CBE53282BF2@codespeak.net> Author: afa Date: Fri Apr 30 18:03:41 2010 New Revision: 74294 Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: This test actually passes, when the typo is fixed. Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 18:03:41 2010 @@ -568,12 +568,11 @@ raises(SystemError, module.clear) def test_new_exception(self): - skip("not working yet") mod = self.import_extension('foo', [ ('newexc', 'METH_VARARGS', ''' char *name = PyString_AsString(PyTuple_GetItem(args, 0)); - return PyExc_NewException(name, PyTuple_GetItem(args, 1), + return PyErr_NewException(name, PyTuple_GetItem(args, 1), PyTuple_GetItem(args, 2)); ''' ), From afa at codespeak.net Fri Apr 30 18:26:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 18:26:55 +0200 (CEST) Subject: [pypy-svn] r74295 - pypy/branch/py-packagecontext/pypy/module/cpyext/test Message-ID: <20100430162655.D2221282BF2@codespeak.net> Author: afa Date: Fri Apr 30 18:26:54 2010 New Revision: 74295 Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Log: Fix the refleaks in the test: - PyImport_ImportModule returns a new reference which must be released, - on Windows, "import os" allocates some extra memory ("CryptProvider"), so add a first import in setup_class. Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/date.c Fri Apr 30 18:26:54 2010 @@ -6,6 +6,8 @@ void initdate(void) { + PyObject *module; Py_InitModule("date", date_functions); - PyImport_ImportModule("apple.banana"); + module = PyImport_ImportModule("apple.banana"); + Py_DECREF(module); } Modified: pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/py-packagecontext/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 18:26:54 2010 @@ -133,6 +133,8 @@ def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) cls.space.getbuiltinmodule("cpyext") + from pypy.module.imp.importing import importhook + importhook(cls.space, "os") # warm up reference counts def compile_module(self, name, **kwds): state = self.space.fromcache(State) @@ -360,7 +362,6 @@ def test_recursive_package_import(self): - skip("not yet") """ If `cherry.date` is an extension module which imports `apple.banana`, the latter is added to `sys.modules` for the `"apple.banana"` key. From afa at codespeak.net Fri Apr 30 18:41:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 18:41:49 +0200 (CEST) Subject: [pypy-svn] r74296 - in pypy/trunk/pypy: module/cpyext/test rpython/lltypesystem Message-ID: <20100430164149.BFD00282BF2@codespeak.net> Author: afa Date: Fri Apr 30 18:41:48 2010 New Revision: 74296 Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py pypy/trunk/pypy/rpython/lltypesystem/lltype.py Log: Fix apparently leaking tests Modified: pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_unicodeobject.py Fri Apr 30 18:41:48 2010 @@ -115,7 +115,6 @@ rffi.free_charp(b_encoding) def test_leak(self): - py.test.skip("This test seems to leak memory") size = 50 raw_buf, gc_buf = rffi.alloc_buffer(size) for i in range(size): raw_buf[i] = 'a' @@ -133,7 +132,3 @@ rffi.free_wcharp(wbuf) assert space.type(w_str) is space.w_str assert space.str_w(w_str) == "abc?" - - # XXX this test seems to leak references, see test_leak above - from pypy.module.cpyext.test.test_cpyext import freeze_refcnts - freeze_refcnts(self) Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Fri Apr 30 18:41:48 2010 @@ -1452,7 +1452,8 @@ first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: - value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld) + value = _array(typ, n, initialization=initialization, parent=self, parentindex=fld, + track_allocation=track_allocation) else: value = typ._allocate(initialization=initialization, parent=self, parentindex=fld) setattr(self, fld, value) From afa at codespeak.net Fri Apr 30 19:36:20 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 30 Apr 2010 19:36:20 +0200 (CEST) Subject: [pypy-svn] r74297 - in pypy/trunk/pypy: module/cpyext/test rpython/lltypesystem rpython/lltypesystem/test Message-ID: <20100430173620.9055F282BF5@codespeak.net> Author: afa Date: Fri Apr 30 19:36:18 2010 New Revision: 74297 Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/rpython/lltypesystem/lltype.py pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py Log: Start a slightly better interface to the leak detection in lltype. +some unit tests Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Apr 30 19:36:18 2010 @@ -29,15 +29,6 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def set_difference(id_dict1, id_dict2): - d = id_dict1.copy() - for key in id_dict2.keys(): - try: - del d[key] - except KeyError: - pass - return d - class AppTestApi: def setup_class(cls): cls.space = gettestobjspace(usemodules=['cpyext', 'thread']) @@ -75,8 +66,7 @@ self.frozen_refcounts[w_obj] = obj.c_ob_refcnt #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) - self.frozen_lltallocations = lltype.ALLOCATED.copy() - lltype.TRACK_ALLOCATIONS = True + lltype.start_tracking_allocations() class LeakCheckingTest(object): def check_and_print_leaks(self): @@ -121,12 +111,13 @@ leaking = True print >>sys.stderr, "Did not deallocate %r (ll2ctypes)" % (llvalue, ) print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - for llvalue in set_difference(lltype.ALLOCATED, self.frozen_lltallocations).keys(): + for llvalue in lltype.ALLOCATED.keys(): leaking = True print >>sys.stderr, "Did not deallocate %r (llvalue)" % (llvalue, ) print >>sys.stderr, "\t" + "\n\t".join(llvalue._traceback.splitlines()) - return leaking + lltype.stop_tracking_allocations() + return leaking class AppTestCpythonExtensionBase(LeakCheckingTest): def setup_class(cls): Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Fri Apr 30 19:36:18 2010 @@ -16,7 +16,20 @@ import weakref TLS = tlsobject() + +# Track allocations to detect memory leaks +# Don't track 'gc' and immortal mallocs TRACK_ALLOCATIONS = False +ALLOCATED = identity_dict() + +def start_tracking_allocations(): + global TRACK_ALLOCATIONS + TRACK_ALLOCATIONS = True + ALLOCATED.clear() + +def stop_tracking_allocations(): + global TRACK_ALLOCATIONS + TRACK_ALLOCATIONS = False class _uninitialized(object): def __init__(self, TYPE): @@ -1318,8 +1331,6 @@ def _was_freed(self): return False -ALLOCATED = identity_dict() - class _parentable(_container): _kind = "?" Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_lltype.py Fri Apr 30 19:36:18 2010 @@ -1,5 +1,6 @@ import py from pypy.rpython.lltypesystem.lltype import * +from pypy.rpython.lltypesystem import lltype, rffi from pypy.lib.identity_dict import identity_dict def isweak(p, T): @@ -777,3 +778,42 @@ assert hash1 == identityhash(a) p = cast_opaque_ptr(llmemory.GCREF, a) assert hash1 == identityhash(p) + +class TestTrackAllocation: + def setup_method(self, func): + start_tracking_allocations() + + def teardown_method(self, func): + assert not lltype.ALLOCATED, "Memory was not correctly freed" + stop_tracking_allocations() + + def test_track_allocation(self): + """A malloc'd buffer fills the ALLOCATED dictionary""" + assert lltype.TRACK_ALLOCATIONS + assert not lltype.ALLOCATED + buf = malloc(Array(Signed), 1, flavor="raw") + assert len(lltype.ALLOCATED) == 1 + assert lltype.ALLOCATED.keys() == [buf._obj] + free(buf, flavor="raw") + assert not lltype.ALLOCATED + + def test_str_from_buffer(self): + """gc-managed memory does not need to be freed""" + size = 50 + raw_buf, gc_buf = rffi.alloc_buffer(size) + for i in range(size): raw_buf[i] = 'a' + rstr = rffi.str_from_buffer(raw_buf, gc_buf, size, size) + rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + assert not lltype.ALLOCATED + + def test_leak_traceback(self): + """Test info stored for allocated items""" + buf = malloc(Array(Signed), 1, flavor="raw") + traceback = lltype.ALLOCATED.keys()[0]._traceback + lines = traceback.splitlines() + assert 'malloc(' in lines[-1] and 'flavor="raw")' in lines[-1] + + # XXX The traceback should not be too long + print traceback + + free(buf, flavor="raw") From hpk at codespeak.net Fri Apr 30 20:20:51 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 20:20:51 +0200 (CEST) Subject: [pypy-svn] r74298 - in pypy/branch/py12/pypy: jit/backend/cli/test jit/backend/llvm/test lib/app_test/ctypes_tests module/select rlib/rsdl/test rpython/numpy/test translator/benchmark Message-ID: <20100430182051.52B86282BF5@codespeak.net> Author: hpk Date: Fri Apr 30 20:20:45 2010 New Revision: 74298 Modified: pypy/branch/py12/pypy/jit/backend/cli/test/conftest.py pypy/branch/py12/pypy/jit/backend/llvm/test/conftest.py pypy/branch/py12/pypy/lib/app_test/ctypes_tests/conftest.py pypy/branch/py12/pypy/module/select/conftest.py pypy/branch/py12/pypy/rlib/rsdl/test/conftest.py pypy/branch/py12/pypy/rpython/numpy/test/conftest.py pypy/branch/py12/pypy/translator/benchmark/conftest.py Log: simplify directory/conditional skipping and get get rid of deprecations Modified: pypy/branch/py12/pypy/jit/backend/cli/test/conftest.py ============================================================================== --- pypy/branch/py12/pypy/jit/backend/cli/test/conftest.py (original) +++ pypy/branch/py12/pypy/jit/backend/cli/test/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,5 +1,4 @@ import py -class Directory(py.test.collect.Directory): - def collect(self): - py.test.skip("CLI backend tests skipped for now") +def pytest_collect_directory(path): + py.test.skip("CLI backend tests skipped for now") Modified: pypy/branch/py12/pypy/jit/backend/llvm/test/conftest.py ============================================================================== --- pypy/branch/py12/pypy/jit/backend/llvm/test/conftest.py (original) +++ pypy/branch/py12/pypy/jit/backend/llvm/test/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,5 +1,4 @@ import py -class Directory(py.test.collect.Directory): - def collect(self): - py.test.skip("llvm backend tests skipped for now") +def pytest_collect_directory(): + py.test.skip("llvm backend tests skipped for now") Modified: pypy/branch/py12/pypy/lib/app_test/ctypes_tests/conftest.py ============================================================================== --- pypy/branch/py12/pypy/lib/app_test/ctypes_tests/conftest.py (original) +++ pypy/branch/py12/pypy/lib/app_test/ctypes_tests/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,11 +1,9 @@ import py import sys -class Directory(py.test.collect.Directory): - def collect(self): - if '__pypy__' not in sys.builtin_module_names: - py.test.skip("these tests are meant to be run on top of pypy-c") - return super(Directory, self).collect() +def pytest_collect_directory(): + if '__pypy__' not in sys.builtin_module_names: + py.test.skip("these tests are meant to be run on top of pypy-c") def compile_so_file(): from pypy.translator.platform import platform Modified: pypy/branch/py12/pypy/module/select/conftest.py ============================================================================== --- pypy/branch/py12/pypy/module/select/conftest.py (original) +++ pypy/branch/py12/pypy/module/select/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,10 +1,4 @@ import py -class Directory(py.test.collect.Directory): - - def run(self): - try: - import ctypes - except ImportError: - py.test.skip("these tests need ctypes installed") - return super(Directory, self).run() +def pytest_collect_directory(): + py.test.importorskip("ctypes") Modified: pypy/branch/py12/pypy/rlib/rsdl/test/conftest.py ============================================================================== --- pypy/branch/py12/pypy/rlib/rsdl/test/conftest.py (original) +++ pypy/branch/py12/pypy/rlib/rsdl/test/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,10 +1,8 @@ from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled import py -class Directory(py.test.collect.Directory): - def collect(self): - try: - check_sdl_installation() - except SDLNotInstalled, e: - py.test.skip("SDL not installed(?): %s" % (e,)) - return py.test.collect.Directory.collect(self) +def pytest_collect_directory(): + try: + check_sdl_installation() + except SDLNotInstalled, e: + py.test.skip("SDL not installed(?): %s" % (e,)) Modified: pypy/branch/py12/pypy/rpython/numpy/test/conftest.py ============================================================================== --- pypy/branch/py12/pypy/rpython/numpy/test/conftest.py (original) +++ pypy/branch/py12/pypy/rpython/numpy/test/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,10 +1,4 @@ import py -class Directory(py.test.collect.Directory): - - def run(self): - try: - import numpy - except ImportError: - py.test.skip("these tests need numpy installed") - return super(Directory, self).run() +def pytest_collect_directory(): + py.test.importorskip("numpy") Modified: pypy/branch/py12/pypy/translator/benchmark/conftest.py ============================================================================== --- pypy/branch/py12/pypy/translator/benchmark/conftest.py (original) +++ pypy/branch/py12/pypy/translator/benchmark/conftest.py Fri Apr 30 20:20:45 2010 @@ -1,7 +1,4 @@ import py -class Directory(py.test.collect.Directory): - - def recfilter(self, path): - # exclude the subdirectories added by 'svn co' from benchmarks.py - return path.check(basename='test') +def pytest_ignore_collect_path(path): + return path.basename == "test" From cfbolz at codespeak.net Fri Apr 30 20:33:32 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 30 Apr 2010 20:33:32 +0200 (CEST) Subject: [pypy-svn] r74299 - pypy/extradoc/planning Message-ID: <20100430183332.190B6282BF5@codespeak.net> Author: cfbolz Date: Fri Apr 30 20:33:28 2010 New Revision: 74299 Modified: pypy/extradoc/planning/jit.txt Log: (arigo, cfbolz): some things we came up with in the pub Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Fri Apr 30 20:33:28 2010 @@ -1,14 +1,27 @@ +NEW TASKS +--------- + +- look at assembler-assembler calls again: if the inner function is traced + after the outer one, the call is slow. other cases could be faster too, + probably. + +- have benchmarks for jit compile time and jit memory usage + +- trace into functions even if they have a loop. only if the loop is actually + hit, a residual portal call is produced + +- generators are not really fast ? maybe add a JUMP_ABSOLUTE_GENERATOR that + does not call can_enter_jit after an iteration in which there was a yield. + obviously. + + TASKS ----- -- sort out a benchmark infrastructure. graphs! - - think about code memory management - forcing virtualizables should only force fields affected, not everything -- compress resume data (?) - - think out looking into functions or not, based on arguments, for example contains__Tuple should be unrolled if tuple is of constant length. HARD, blocked by the fact that we don't know constants soon enough From hpk at codespeak.net Fri Apr 30 20:48:11 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 20:48:11 +0200 (CEST) Subject: [pypy-svn] r74300 - pypy/branch/py12/pypy/module/oracle/test Message-ID: <20100430184811.CBAD4282BF5@codespeak.net> Author: hpk Date: Fri Apr 30 20:48:08 2010 New Revision: 74300 Added: pypy/branch/py12/pypy/module/oracle/test/conftest.py Modified: pypy/branch/py12/pypy/module/oracle/test/test_connect.py Log: shift the check for oracle importability into a more general place, issue being: starting with the 1.2 series py.test follows the convention (as discussed on the testing-in-python mailing list) of always calling a teardown_X even if the corresponding setup_X failed. Reason being that setup functions often setup various resources and a partial fail will leave some resources unfinished. calling teardown gives a chance to properly tear down the partially setup resources. Added: pypy/branch/py12/pypy/module/oracle/test/conftest.py ============================================================================== --- (empty file) +++ pypy/branch/py12/pypy/module/oracle/test/conftest.py Fri Apr 30 20:48:08 2010 @@ -0,0 +1,3 @@ +import py +def pytest_runtest_setup(): + py.test.importorskip("pypy.module.oracle.roci") Modified: pypy/branch/py12/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/branch/py12/pypy/module/oracle/test/test_connect.py (original) +++ pypy/branch/py12/pypy/module/oracle/test/test_connect.py Fri Apr 30 20:48:08 2010 @@ -7,11 +7,6 @@ @classmethod def setup_class(cls): - try: - from pypy.module.oracle import roci - except ImportError: - py.test.skip("Oracle client not available") - space = gettestobjspace(usemodules=('oracle',)) cls.space = space space.setitem(space.builtin.w_dict, space.wrap('oracle'), From hpk at codespeak.net Fri Apr 30 21:39:22 2010 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 30 Apr 2010 21:39:22 +0200 (CEST) Subject: [pypy-svn] r74301 - pypy/branch/py12/pypy/rpython/numpy/test Message-ID: <20100430193922.CC64F282BF5@codespeak.net> Author: hpk Date: Fri Apr 30 21:39:20 2010 New Revision: 74301 Removed: pypy/branch/py12/pypy/rpython/numpy/test/conftest.py Modified: pypy/branch/py12/pypy/rpython/numpy/test/test_array.py Log: simplify import skipping here Modified: pypy/branch/py12/pypy/rpython/numpy/test/test_array.py ============================================================================== --- pypy/branch/py12/pypy/rpython/numpy/test/test_array.py (original) +++ pypy/branch/py12/pypy/rpython/numpy/test/test_array.py Fri Apr 30 21:39:20 2010 @@ -3,6 +3,7 @@ """ import py +numpy = py.test.importorskip("numpy") import pypy.rpython.numpy.implementation from pypy.annotation import model as annmodel from pypy.annotation.model import SomeObject, SomeInteger, SomeChar, SomeTuple @@ -15,13 +16,6 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.rint import IntegerRepr -def setup_module(mod): - try: - import numpy - except ImportError: - py.test.skip("numpy not found") - mod.numpy = numpy - from pypy.rpython.numpy.rarray import ArrayRepr from pypy.rpython.numpy.aarray import SomeArray