From pypy.commits at gmail.com Thu Feb 1 18:32:14 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 01 Feb 2018 15:32:14 -0800 (PST) Subject: [pypy-commit] pypy vmprof-resolve_addr: a branch where to implement _vmprof.resolve_addr, which seems to be needed to properly handle native frames nowadays Message-ID: <5a73a37e.cb9edf0a.e6163.3af9@mx.google.com> Author: Antonio Cuni Branch: vmprof-resolve_addr Changeset: r93730:0d97421e6dfb Date: 2018-02-02 00:21 +0100 http://bitbucket.org/pypy/pypy/changeset/0d97421e6dfb/ Log: a branch where to implement _vmprof.resolve_addr, which seems to be needed to properly handle native frames nowadays From pypy.commits at gmail.com Thu Feb 1 18:32:18 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 01 Feb 2018 15:32:18 -0800 (PST) Subject: [pypy-commit] pypy vmprof-resolve_addr: expose rvmprof.resolve_addr as _vmprof.resolve_addr Message-ID: <5a73a382.83871c0a.28618.0c2f@mx.google.com> Author: Antonio Cuni Branch: vmprof-resolve_addr Changeset: r93732:aef1d565d8b2 Date: 2018-02-02 00:31 +0100 http://bitbucket.org/pypy/pypy/changeset/aef1d565d8b2/ Log: expose rvmprof.resolve_addr as _vmprof.resolve_addr diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -17,6 +17,7 @@ 'get_profile_path': 'interp_vmprof.get_profile_path', 'stop_sampling': 'interp_vmprof.stop_sampling', 'start_sampling': 'interp_vmprof.start_sampling', + 'resolve_addr': 'interp_vmprof.resolve_addr', 'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError', } diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.baseobjspace import W_Root from rpython.rlib import rvmprof, jit +from rpython.rtyper.lltypesystem import rffi from pypy.interpreter.error import oefmt # ____________________________________________________________ @@ -98,3 +99,19 @@ def start_sampling(space): rvmprof.start_sampling() return space.w_None + + at unwrap_spec(addr=int) +def resolve_addr(space, addr): + addr = rffi.cast(rffi.VOIDP, addr) + try: + name, lineno, srcfile = rvmprof.resolve_addr(addr) + except ValueError: + name = '' + lineno = 0 + srcfile = '' + # + w_name = space.newtext(name) + w_lineno = space.newint(lineno) + w_srcfile = space.newtext(srcfile) + return space.newtuple([w_name, w_lineno, w_srcfile]) + diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -150,3 +150,10 @@ assert pos3 > pos _vmprof.disable() + def test_resolve_addr(self): + import _vmprof + addr = 0x517450 # XXX dont' hardcode + name, lineno, srcfile = _vmprof.resolve_addr(addr) + assert name == 'PyLong_AsLong' + assert lineno == 0 + assert srcfile == 'python' From pypy.commits at gmail.com Thu Feb 1 18:32:16 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 01 Feb 2018 15:32:16 -0800 (PST) Subject: [pypy-commit] pypy vmprof-resolve_addr: WIP: expose vmp_resolve_addr in rlib.rvmprof Message-ID: <5a73a380.1488df0a.43b0f.42a5@mx.google.com> Author: Antonio Cuni Branch: vmprof-resolve_addr Changeset: r93731:55632ddaa2c1 Date: 2018-02-02 00:22 +0100 http://bitbucket.org/pypy/pypy/changeset/55632ddaa2c1/ Log: WIP: expose vmp_resolve_addr in rlib.rvmprof diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -61,6 +61,9 @@ def start_sampling(): return _get_vmprof().start_sampling() +def resolve_addr(addr): + return _get_vmprof().resolve_addr(addr) + # ---------------- # stacklet support # ---------------- @@ -80,3 +83,4 @@ def restore_stack(x): vmprof_tl_stack.setraw(x) start_sampling() + diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -146,6 +146,11 @@ lltype.Void, compilation_info=eci, _nowrapper=True) + vmp_resolve_addr = rffi.llexternal("vmp_resolve_addr", + [rffi.VOIDP, rffi.CCHARP, rffi.INT, + rffi.INTP, rffi.CCHARP, rffi.INT], + rffi.INT, compilation_info=eci) + return CInterface(locals()) diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -186,6 +186,26 @@ """ self.cintf.vmprof_start_sampling() + def resolve_addr(self, addr): + name_len = 128 + srcfile_len = 256 + with lltype.scoped_alloc(rffi.CCHARP.TO, name_len) as name_p: + with lltype.scoped_alloc(rffi.CCHARP.TO, srcfile_len) as srcfile_p: + with lltype.scoped_alloc(rffi.INTP.TO, 1) as lineno_p: + # XXX vmp_resolve_addr checks whether the first char is 0 + # before calling dladdr, not sure why. Investigate before + # merging. + name_p[0] = '\0' + srcfile_p[0] = '\0' + res = self.cintf.vmp_resolve_addr(addr, name_p, name_len, + lineno_p, srcfile_p, srcfile_len) + if res != 0: + raise ValueError("Cannot resolve name") + # + name = rffi.charp2strn(name_p, name_len) + srcfile = rffi.charp2strn(srcfile_p, srcfile_len) + lineno = lineno_p[0] + return name, lineno, srcfile def vmprof_execute_code(name, get_code_fn, result_class=None, _hack_update_stack_untranslated=False): diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -41,6 +41,9 @@ RPY_EXTERN int vmprof_stop_sampling(void); RPY_EXTERN void vmprof_start_sampling(void); +RPY_EXTERN int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, + char * srcfile, int srcfile_len); + long vmprof_write_header_for_jit_addr(intptr_t *result, long n, intptr_t addr, int max_depth); diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -199,3 +199,13 @@ del not_found[i] break assert not_found == [] + + +class TestSymbolTable(object): + + def test_vmp_resolve_addr(self): + # XXX: don't hardcode this addr + addr = rffi.cast(rffi.VOIDP, 0x517450) + name, lineno, srcfile = rvmprof.resolve_addr(addr) + assert name == 'PyLong_AsLong' + From pypy.commits at gmail.com Fri Feb 2 07:38:18 2018 From: pypy.commits at gmail.com (barrywhart) Date: Fri, 02 Feb 2018 04:38:18 -0800 (PST) Subject: [pypy-commit] pypy 2634_datetime_timedelta_performance: Fix accidental change Message-ID: <5a745bba.74acdf0a.ceffe.8603@mx.google.com> Author: Barry Hart Branch: 2634_datetime_timedelta_performance Changeset: r93735:ac4aeeed1bd1 Date: 2018-01-01 08:21 -0500 http://bitbucket.org/pypy/pypy/changeset/ac4aeeed1bd1/ Log: Fix accidental change diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1540,7 +1540,7 @@ offset = self._utcoffset() if offset: # neither None nor 0 mm -= offset - y, m, d, hh, mm, ss, _ = normalize_datetime( + y, m, d, hh, mm, ss, _ = _normalize_datetime( y, m, d, hh, mm, ss, 0, ignore_overflow=True) return _build_struct_time(y, m, d, hh, mm, ss, 0) From pypy.commits at gmail.com Fri Feb 2 07:38:13 2018 From: pypy.commits at gmail.com (barrywhart) Date: Fri, 02 Feb 2018 04:38:13 -0800 (PST) Subject: [pypy-commit] pypy 2634_datetime_timedelta_performance: Implement __iadd__ and __isub__ on datetime objects Message-ID: <5a745bb5.f487df0a.69fbc.11b6@mx.google.com> Author: Barry Hart Branch: 2634_datetime_timedelta_performance Changeset: r93733:6513e6af0511 Date: 2017-12-31 21:40 -0500 http://bitbucket.org/pypy/pypy/changeset/6513e6af0511/ Log: Implement __iadd__ and __isub__ on datetime objects diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1815,6 +1815,29 @@ __radd__ = __add__ + def _iadd_timedelta(self, other, factor): + if not isinstance(other, timedelta): + return NotImplemented + self._year, self._month, self._day, self._hour, self._minute, \ + self._second, self._microsecond = _normalize_datetime( + self._year, + self._month, + self._day + other.days * factor, + self._hour, + self._minute, + self._second + other.seconds * factor, + self._microsecond + other.microseconds * factor) + + def __iadd__(self, other): + "Increment a datetime by a timedelta." + self._iadd_timedelta(other, 1) + return self + + def __isub__(self, other): + "Decrement a datetime by a timedelta." + self._iadd_timedelta(other, -1) + return self + def __sub__(self, other): "Subtract two datetimes, or a datetime and a timedelta." if not isinstance(other, datetime): From pypy.commits at gmail.com Fri Feb 2 07:38:20 2018 From: pypy.commits at gmail.com (barrywhart) Date: Fri, 02 Feb 2018 04:38:20 -0800 (PST) Subject: [pypy-commit] pypy 2634_datetime_timedelta_performance: Use new construction method in _from_timestamp Message-ID: <5a745bbc.4d051c0a.101cb.8f4c@mx.google.com> Author: Barry Hart Branch: 2634_datetime_timedelta_performance Changeset: r93736:eed3086fd841 Date: 2018-01-01 08:32 -0500 http://bitbucket.org/pypy/pypy/changeset/eed3086fd841/ Log: Use new construction method in _from_timestamp diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1497,7 +1497,7 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): From pypy.commits at gmail.com Fri Feb 2 07:38:16 2018 From: pypy.commits at gmail.com (barrywhart) Date: Fri, 02 Feb 2018 04:38:16 -0800 (PST) Subject: [pypy-commit] pypy 2634_datetime_timedelta_performance: Remove __iadd__ and __isub__ functions. Add streamlined path in datetime.__new__. Message-ID: <5a745bb8.5088df0a.2fc51.d69b@mx.google.com> Author: Barry Hart Branch: 2634_datetime_timedelta_performance Changeset: r93734:9aba94e5fe7c Date: 2018-01-01 08:15 -0500 http://bitbucket.org/pypy/pypy/changeset/9aba94e5fe7c/ Log: Remove __iadd__ and __isub__ functions. Add streamlined path in datetime.__new__. diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1412,6 +1412,15 @@ self.__setstate(year, month) self._hashcode = -1 return self + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + self = object.__new__(cls) + self._year, self._month, self._day, self._hour, self._minute, \ + self._second, self._microsecond = year + self._tzinfo = tzinfo + self._hashcode = -1 + return self year, month, day = _check_date_fields(year, month, day) hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) @@ -1531,7 +1540,7 @@ offset = self._utcoffset() if offset: # neither None nor 0 mm -= offset - y, m, d, hh, mm, ss, _ = _normalize_datetime( + y, m, d, hh, mm, ss, _ = normalize_datetime( y, m, d, hh, mm, ss, 0, ignore_overflow=True) return _build_struct_time(y, m, d, hh, mm, ss, 0) @@ -1797,7 +1806,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1805,7 +1814,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." @@ -1815,29 +1824,6 @@ __radd__ = __add__ - def _iadd_timedelta(self, other, factor): - if not isinstance(other, timedelta): - return NotImplemented - self._year, self._month, self._day, self._hour, self._minute, \ - self._second, self._microsecond = _normalize_datetime( - self._year, - self._month, - self._day + other.days * factor, - self._hour, - self._minute, - self._second + other.seconds * factor, - self._microsecond + other.microseconds * factor) - - def __iadd__(self, other): - "Increment a datetime by a timedelta." - self._iadd_timedelta(other, 1) - return self - - def __isub__(self, other): - "Decrement a datetime by a timedelta." - self._iadd_timedelta(other, -1) - return self - def __sub__(self, other): "Subtract two datetimes, or a datetime and a timedelta." if not isinstance(other, datetime): From pypy.commits at gmail.com Fri Feb 2 07:38:22 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 02 Feb 2018 04:38:22 -0800 (PST) Subject: [pypy-commit] pypy 2634_datetime_timedelta_performance: modify __new__ less invasively Message-ID: <5a745bbe.65a8df0a.95b13.1344@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: 2634_datetime_timedelta_performance Changeset: r93737:2a0f482a854d Date: 2018-02-02 13:33 +0100 http://bitbucket.org/pypy/pypy/changeset/2a0f482a854d/ Log: modify __new__ less invasively diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1415,15 +1415,11 @@ elif isinstance(year, tuple) and len(year) == 7: # Used by internal functions where the arguments are guaranteed to # be valid. - self = object.__new__(cls) - self._year, self._month, self._day, self._hour, self._minute, \ - self._second, self._microsecond = year - self._tzinfo = tzinfo - self._hashcode = -1 - return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = object.__new__(cls) self._year = year From pypy.commits at gmail.com Fri Feb 2 07:38:24 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 02 Feb 2018 04:38:24 -0800 (PST) Subject: [pypy-commit] pypy default: merge 2634_datetime_timedelta_performance Message-ID: <5a745bc0.c7a9df0a.1de89.88ff@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93738:2f5eb5410769 Date: 2018-02-02 13:36 +0100 http://bitbucket.org/pypy/pypy/changeset/2f5eb5410769/ Log: merge 2634_datetime_timedelta_performance improve the performance of datetime + timedelta by skipping the consistency checks of the datetime values (they are correct by construction) This fixes #2634 Thanks Barry Whart! diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1415,9 +1415,14 @@ self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = dateinterop.__new__(cls) self._year = year @@ -1491,7 +1496,7 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): @@ -1800,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1808,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." From pypy.commits at gmail.com Fri Feb 2 16:25:35 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 02 Feb 2018 13:25:35 -0800 (PST) Subject: [pypy-commit] pypy default: merge cpyext-faster-arg-passing Message-ID: <5a74d74f.49a0df0a.69728.4000@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93739:6a370242b86c Date: 2018-02-02 22:24 +0100 http://bitbucket.org/pypy/pypy/changeset/6a370242b86c/ Log: merge cpyext-faster-arg-passing When using cpyext, improve the speed of passing certain objects from PyPy to C code, most notably None, True, False, types, all instances of C-defined types. Before, a dict lookup was needed every time such an object crossed over, now it is just a field read. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -23,3 +23,11 @@ added, then the performance using mapdict is linear in the number of attributes. This is now fixed (by switching to a regular dict after 80 attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,6 +208,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,10 +3,20 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -157,6 +167,25 @@ assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + def test_shortcut(self): + # test that instances of classes that are defined in C become an + # instance of W_BaseCPyObject and thus can be converted faster back to + # their pyobj, because they store a pointer to it directly. + if self.runappdirect: + skip("can't run with -A") + module = self.import_module(name='foo') + obj = module.fooType() + assert self._check_uses_shortcut(obj) + # W_TypeObjects use shortcut + assert self._check_uses_shortcut(object) + assert self._check_uses_shortcut(type) + # None, True, False use shortcut + assert self._check_uses_shortcut(None) + assert self._check_uses_shortcut(True) + assert self._check_uses_shortcut(False) + assert not self._check_uses_shortcut(1) + assert not self._check_uses_shortcut(object()) + def test_multiple_inheritance1(self): module = self.import_module(name='foo') obj = module.UnicodeSubtype(u'xyz') From pypy.commits at gmail.com Sat Feb 3 05:59:37 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 03 Feb 2018 02:59:37 -0800 (PST) Subject: [pypy-commit] pypy default: document branch Message-ID: <5a759619.05a4df0a.11333.1413@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93741:516a8cd83164 Date: 2018-02-03 11:58 +0100 http://bitbucket.org/pypy/pypy/changeset/516a8cd83164/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -31,3 +31,8 @@ code, most notably None, True, False, types, all instances of C-defined types. Before, a dict lookup was needed every time such an object crossed over, now it is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. From pypy.commits at gmail.com Sat Feb 3 05:59:35 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 03 Feb 2018 02:59:35 -0800 (PST) Subject: [pypy-commit] pypy default: set the hypothesis deadline of this test explicitly to None (it's slow, because Message-ID: <5a759617.4dc5df0a.7336f.9d5d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93740:d1c1e6659eab Date: 2018-02-03 11:58 +0100 http://bitbucket.org/pypy/pypy/changeset/d1c1e6659eab/ Log: set the hypothesis deadline of this test explicitly to None (it's slow, because looking up the codec the first time is slow, not anything related to the test). It will still be stopped after less than 100 examples by the overall timeout of 60s diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) From pypy.commits at gmail.com Sat Feb 3 17:23:13 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 03 Feb 2018 14:23:13 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: be a lot more systematic about testing the rbigint.int_* variants. This Message-ID: <5a763651.0dc4df0a.cb38b.5be5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93743:fb3f2424ac23 Date: 2018-02-03 23:22 +0100 http://bitbucket.org/pypy/pypy/changeset/fb3f2424ac23/ Log: be a lot more systematic about testing the rbigint.int_* variants. This discovered a bug in rbigint.int_divmod (see failing test) diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -17,15 +17,31 @@ from hypothesis import given, strategies +def gen_signs(l): + for s in l: + if s == 0: + yield s + else: + yield s + yield -s + long_vals_not_too_big = range(17) + [ - 37, 50, + 37, 39, 50, 127, 128, 129, 511, 512, 513, sys.maxint, sys.maxint + 1, + 12345678901234567890L, 123456789123456789000000L, - ] +] long_vals = long_vals_not_too_big + [ 1 << 100, 3 ** 10000] +int_vals = range(33) + [ + 1000, + 0x11111111, 0x11111112, 8888, + 9999, sys.maxint, 2 ** 19, 2 ** 18 - 1 +] +signed_int_vals = list(gen_signs(int_vals)) + [-sys.maxint-1] + class TestRLong(object): def test_simple(self): for op1 in [-2, -1, 0, 1, 2, 10, 50]: @@ -71,22 +87,24 @@ assert r2.tolong() == 100L for op1 in gen_signs(long_vals): - for op2 in gen_signs(long_vals): - if not op2 or op2 >= (1 << SHIFT) or op2 <= -(1 << SHIFT): + for op2 in signed_int_vals: + if not op2: continue rl_op1 = rbigint.fromlong(op1) r1 = rl_op1.int_floordiv(op2) r2 = op1 // op2 assert r1.tolong() == r2 - + assert py.test.raises(ZeroDivisionError, r.int_floordiv, 0) # Error pointed out by Armin Rigo n = sys.maxint+1 r = rbigint.fromlong(n) assert r.int_floordiv(int(-n)).tolong() == -1L - - for x in (1, 1000, sys.maxint): + + for x in int_vals: + if not x: + continue r = rbigint.fromlong(x) rn = rbigint.fromlong(-x) res = r.int_floordiv(x) @@ -94,7 +112,7 @@ res3 = rn.int_floordiv(x) assert res.tolong() == 1L assert res2.tolong() == -1L - assert res3.tolong() == -1L + assert res3.tolong() == -1L def test_truediv(self): for op1 in gen_signs(long_vals_not_too_big): @@ -153,12 +171,14 @@ rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.mod(rl_op2) r2 = op1 % op2 - + assert r1.tolong() == r2 def test_int_mod(self): for x in gen_signs(long_vals): - for y in gen_signs([1, 2, 4, 8, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: + if not y: + continue op1 = rbigint.fromlong(x) r1 = op1.int_mod(y) r2 = x % y @@ -212,13 +232,6 @@ assert not (a1 == a3) -def gen_signs(l): - for s in l: - if s == 0: - yield s - else: - yield s - yield -s def bigint(lst, sign): for digit in lst: @@ -320,7 +333,7 @@ def test_int_add(self): for x in gen_signs(long_vals): - for y in gen_signs([0, 1, 9999, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_add(y) assert result.tolong() == x + y @@ -337,7 +350,7 @@ def test_int_sub(self): for x in gen_signs([0, 123456789123456789000000L, 1 << 100, 3 ** 10000]): - for y in gen_signs([0, 1, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_sub(y) assert result.tolong() == x - y @@ -358,8 +371,8 @@ assert result.tolong() == x * x def test_int_mul(self): - for x in gen_signs([39, 128, 111111111, 123456789123456789000000L, 1 << 100, 3 ** 10000]): - for y in gen_signs([0, 1, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for x in gen_signs(long_vals): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_mul(y) assert result.tolong() == x * y @@ -442,14 +455,14 @@ def test_int_comparison(self): for x in gen_signs(long_vals): - for y in gen_signs([0, 1, 0x11111111, 0x11111112, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) - assert (x < y) == f1.int_lt(y) - assert (x <= y) == f1.int_le(y) - assert (x > y) == f1.int_gt(y) - assert (x >= y) == f1.int_ge(y) - assert (x == y) == f1.int_eq(y) - assert (x != y) == f1.int_ne(y) + assert (x < y) == f1.int_lt(y) + assert (x <= y) == f1.int_le(y) + assert (x > y) == f1.int_gt(y) + assert (x >= y) == f1.int_ge(y) + assert (x == y) == f1.int_eq(y) + assert (x != y) == f1.int_ne(y) def test_order(self): f6 = rbigint.fromint(6) @@ -879,19 +892,24 @@ assert rem.tolong() == _rem def test_int_divmod(self): - x = 12345678901234567890L - for i in range(10): - y = randint(0, 1 << 12) - for sx, sy in (1, 1), (1, -1), (-1, -1), (-1, 1): - sx *= x - sy *= y - f1 = rbigint.fromlong(sx) - div, rem = f1.int_divmod(sy) - _div, _rem = divmod(sx, sy) - print sx, sy, " | ", div.tolong(), rem.tolong() - assert div.tolong() == _div - assert rem.tolong() == _rem - + for x in long_vals: + for y in int_vals + [-sys.maxint-1]: + if not y: + continue + for sx, sy in (1, 1), (1, -1), (-1, -1), (-1, 1): + sx *= x + sy *= y + if sx == 0 and sy == 1: + import pdb; pdb.set_trace() + if sy == sys.maxint + 1: + continue + f1 = rbigint.fromlong(sx) + div, rem = f1.int_divmod(sy) + _div, _rem = divmod(sx, sy) + print sx, sy, " | ", div.tolong(), rem.tolong() + assert div.tolong() == _div + assert rem.tolong() == _rem + # testing Karatsuba stuff def test__v_iadd(self): f1 = bigint([lobj.MASK] * 10, 1) From pypy.commits at gmail.com Sat Feb 3 17:23:11 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 03 Feb 2018 14:23:11 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: move from/tobytes tests to correct class Message-ID: <5a76364f.0dc4df0a.cb38b.5be4@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93742:62e5eec4ea05 Date: 2018-02-03 22:43 +0100 http://bitbucket.org/pypy/pypy/changeset/62e5eec4ea05/ Log: move from/tobytes tests to correct class diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -758,6 +758,47 @@ assert rbigint.fromlong(2).log(2.0) == 1.0 assert rbigint.fromlong(2**1023).log(2.0) == 1023.0 + def test_frombytes(self): + bigint = rbigint.frombytes('', byteorder='big', signed=True) + assert bigint.tolong() == 0 + s = "\xFF\x12\x34\x56" + bigint = rbigint.frombytes(s, byteorder="big", signed=False) + assert bigint.tolong() == 0xFF123456 + bigint = rbigint.frombytes(s, byteorder="little", signed=False) + assert bigint.tolong() == 0x563412FF + s = "\xFF\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\xFF" + bigint = rbigint.frombytes(s, byteorder="big", signed=False) + assert s == bigint.tobytes(16, byteorder="big", signed=False) + py.test.raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', + signed=True) + bigint = rbigint.frombytes('\x82', byteorder='big', signed=True) + assert bigint.tolong() == -126 + + def test_tobytes(self): + assert rbigint.fromint(0).tobytes(1, 'big', signed=True) == '\x00' + assert rbigint.fromint(1).tobytes(2, 'big', signed=True) == '\x00\x01' + py.test.raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) + assert rbigint.fromint(-129).tobytes(2, 'big', signed=True) == '\xff\x7f' + assert rbigint.fromint(-129).tobytes(2, 'little', signed=True) == '\x7f\xff' + assert rbigint.fromint(65535).tobytes(3, 'big', signed=True) == '\x00\xff\xff' + assert rbigint.fromint(-65536).tobytes(3, 'little', signed=True) == '\x00\x00\xff' + assert rbigint.fromint(65535).tobytes(2, 'big', signed=False) == '\xff\xff' + assert rbigint.fromint(-8388608).tobytes(3, 'little', signed=True) == '\x00\x00\x80' + i = rbigint.fromint(-8388608) + py.test.raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) + py.test.raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) + py.test.raises(OverflowError, i.tobytes, 2, 'little', signed=True) + + @given(strategies.binary(), strategies.booleans(), strategies.booleans()) + def test_frombytes_tobytes_hypothesis(self, s, big, signed): + # check the roundtrip from binary strings to bigints and back + byteorder = 'big' if big else 'little' + bigint = rbigint.frombytes(s, byteorder=byteorder, signed=signed) + t = bigint.tobytes(len(s), byteorder=byteorder, signed=signed) + assert s == t + + + class TestInternalFunctions(object): def test__inplace_divrem1(self): # signs are not handled in the helpers! @@ -1028,45 +1069,6 @@ res = interpret(fn, []) assert res == -42.0 - def test_frombytes(self): - bigint = rbigint.frombytes('', byteorder='big', signed=True) - assert bigint.tolong() == 0 - s = "\xFF\x12\x34\x56" - bigint = rbigint.frombytes(s, byteorder="big", signed=False) - assert bigint.tolong() == 0xFF123456 - bigint = rbigint.frombytes(s, byteorder="little", signed=False) - assert bigint.tolong() == 0x563412FF - s = "\xFF\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\xFF" - bigint = rbigint.frombytes(s, byteorder="big", signed=False) - assert s == bigint.tobytes(16, byteorder="big", signed=False) - py.test.raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', - signed=True) - bigint = rbigint.frombytes('\x82', byteorder='big', signed=True) - assert bigint.tolong() == -126 - - def test_tobytes(self): - assert rbigint.fromint(0).tobytes(1, 'big', signed=True) == '\x00' - assert rbigint.fromint(1).tobytes(2, 'big', signed=True) == '\x00\x01' - py.test.raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) - assert rbigint.fromint(-129).tobytes(2, 'big', signed=True) == '\xff\x7f' - assert rbigint.fromint(-129).tobytes(2, 'little', signed=True) == '\x7f\xff' - assert rbigint.fromint(65535).tobytes(3, 'big', signed=True) == '\x00\xff\xff' - assert rbigint.fromint(-65536).tobytes(3, 'little', signed=True) == '\x00\x00\xff' - assert rbigint.fromint(65535).tobytes(2, 'big', signed=False) == '\xff\xff' - assert rbigint.fromint(-8388608).tobytes(3, 'little', signed=True) == '\x00\x00\x80' - i = rbigint.fromint(-8388608) - py.test.raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) - py.test.raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) - py.test.raises(OverflowError, i.tobytes, 2, 'little', signed=True) - - @given(strategies.binary(), strategies.booleans(), strategies.booleans()) - def test_frombytes_tobytes_hypothesis(self, s, big, signed): - # check the roundtrip from binary strings to bigints and back - byteorder = 'big' if big else 'little' - bigint = rbigint.frombytes(s, byteorder=byteorder, signed=signed) - t = bigint.tobytes(len(s), byteorder=byteorder, signed=signed) - assert s == t - class TestTranslated(StandaloneTests): From pypy.commits at gmail.com Sat Feb 3 19:39:09 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 03 Feb 2018 16:39:09 -0800 (PST) Subject: [pypy-commit] pypy py3.5: typo causing str -> tuple Message-ID: <5a76562d.90bbdf0a.7fb7e.d316@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93744:20e293849e67 Date: 2018-02-04 02:38 +0200 http://bitbucket.org/pypy/pypy/changeset/20e293849e67/ Log: typo causing str -> tuple diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py --- a/pypy/module/_multiprocessing/__init__.py +++ b/pypy/module/_multiprocessing/__init__.py @@ -16,4 +16,4 @@ interpleveldefs['recv'] = 'interp_win32_py3.multiprocessing_recv' interpleveldefs['send'] = 'interp_win32_py3.multiprocessing_send' else: - interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink', + interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink' From pypy.commits at gmail.com Sun Feb 4 07:53:02 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:02 -0800 (PST) Subject: [pypy-commit] pypy default: move from/tobytes tests to correct class Message-ID: <5a77022e.05da1c0a.969b6.43b7@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93746:ec3f718627ff Date: 2018-02-03 22:43 +0100 http://bitbucket.org/pypy/pypy/changeset/ec3f718627ff/ Log: move from/tobytes tests to correct class diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -686,6 +686,47 @@ assert rbigint.fromlong(2).log(2.0) == 1.0 assert rbigint.fromlong(2**1023).log(2.0) == 1023.0 + def test_frombytes(self): + bigint = rbigint.frombytes('', byteorder='big', signed=True) + assert bigint.tolong() == 0 + s = "\xFF\x12\x34\x56" + bigint = rbigint.frombytes(s, byteorder="big", signed=False) + assert bigint.tolong() == 0xFF123456 + bigint = rbigint.frombytes(s, byteorder="little", signed=False) + assert bigint.tolong() == 0x563412FF + s = "\xFF\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\xFF" + bigint = rbigint.frombytes(s, byteorder="big", signed=False) + assert s == bigint.tobytes(16, byteorder="big", signed=False) + py.test.raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', + signed=True) + bigint = rbigint.frombytes('\x82', byteorder='big', signed=True) + assert bigint.tolong() == -126 + + def test_tobytes(self): + assert rbigint.fromint(0).tobytes(1, 'big', signed=True) == '\x00' + assert rbigint.fromint(1).tobytes(2, 'big', signed=True) == '\x00\x01' + py.test.raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) + assert rbigint.fromint(-129).tobytes(2, 'big', signed=True) == '\xff\x7f' + assert rbigint.fromint(-129).tobytes(2, 'little', signed=True) == '\x7f\xff' + assert rbigint.fromint(65535).tobytes(3, 'big', signed=True) == '\x00\xff\xff' + assert rbigint.fromint(-65536).tobytes(3, 'little', signed=True) == '\x00\x00\xff' + assert rbigint.fromint(65535).tobytes(2, 'big', signed=False) == '\xff\xff' + assert rbigint.fromint(-8388608).tobytes(3, 'little', signed=True) == '\x00\x00\x80' + i = rbigint.fromint(-8388608) + py.test.raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) + py.test.raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) + py.test.raises(OverflowError, i.tobytes, 2, 'little', signed=True) + + @given(strategies.binary(), strategies.booleans(), strategies.booleans()) + def test_frombytes_tobytes_hypothesis(self, s, big, signed): + # check the roundtrip from binary strings to bigints and back + byteorder = 'big' if big else 'little' + bigint = rbigint.frombytes(s, byteorder=byteorder, signed=signed) + t = bigint.tobytes(len(s), byteorder=byteorder, signed=signed) + assert s == t + + + class TestInternalFunctions(object): def test__inplace_divrem1(self): # signs are not handled in the helpers! @@ -950,45 +991,6 @@ res = interpret(fn, []) assert res == -42.0 - def test_frombytes(self): - bigint = rbigint.frombytes('', byteorder='big', signed=True) - assert bigint.tolong() == 0 - s = "\xFF\x12\x34\x56" - bigint = rbigint.frombytes(s, byteorder="big", signed=False) - assert bigint.tolong() == 0xFF123456 - bigint = rbigint.frombytes(s, byteorder="little", signed=False) - assert bigint.tolong() == 0x563412FF - s = "\xFF\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\xFF" - bigint = rbigint.frombytes(s, byteorder="big", signed=False) - assert s == bigint.tobytes(16, byteorder="big", signed=False) - py.test.raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', - signed=True) - bigint = rbigint.frombytes('\x82', byteorder='big', signed=True) - assert bigint.tolong() == -126 - - def test_tobytes(self): - assert rbigint.fromint(0).tobytes(1, 'big', signed=True) == '\x00' - assert rbigint.fromint(1).tobytes(2, 'big', signed=True) == '\x00\x01' - py.test.raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) - assert rbigint.fromint(-129).tobytes(2, 'big', signed=True) == '\xff\x7f' - assert rbigint.fromint(-129).tobytes(2, 'little', signed=True) == '\x7f\xff' - assert rbigint.fromint(65535).tobytes(3, 'big', signed=True) == '\x00\xff\xff' - assert rbigint.fromint(-65536).tobytes(3, 'little', signed=True) == '\x00\x00\xff' - assert rbigint.fromint(65535).tobytes(2, 'big', signed=False) == '\xff\xff' - assert rbigint.fromint(-8388608).tobytes(3, 'little', signed=True) == '\x00\x00\x80' - i = rbigint.fromint(-8388608) - py.test.raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) - py.test.raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) - py.test.raises(OverflowError, i.tobytes, 2, 'little', signed=True) - - @given(strategies.binary(), strategies.booleans(), strategies.booleans()) - def test_frombytes_tobytes_hypothesis(self, s, big, signed): - # check the roundtrip from binary strings to bigints and back - byteorder = 'big' if big else 'little' - bigint = rbigint.frombytes(s, byteorder=byteorder, signed=signed) - t = bigint.tobytes(len(s), byteorder=byteorder, signed=signed) - assert s == t - class TestTranslated(StandaloneTests): From pypy.commits at gmail.com Sun Feb 4 07:53:00 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:00 -0800 (PST) Subject: [pypy-commit] pypy default: tests for rbigint.ne Message-ID: <5a77022c.5088df0a.2fc51.baa9@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93745:25474c7af9f3 Date: 2018-02-04 12:08 +0100 http://bitbucket.org/pypy/pypy/changeset/25474c7af9f3/ Log: tests for rbigint.ne diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -141,6 +141,7 @@ r2 = op1 ** op2 assert r1.tolong() == r2 + def test_touint(self): result = r_uint(sys.maxint + 42) rl = rbigint.fromint(sys.maxint).add(rbigint.fromint(42)) @@ -358,7 +359,7 @@ null = rbigint.fromfloat(-0.0) assert null.int_eq(0) - def test_eq(self): + def test_eq_ne(self): x = 5858393919192332223L y = 585839391919233111223311112332L f1 = rbigint.fromlong(x) @@ -370,6 +371,12 @@ assert not f1.eq(f2) assert not f1.eq(f3) + assert not f1.ne(f1) + assert not f2.ne(f2) + assert not f3.ne(f3) + assert f1.ne(f2) + assert f1.ne(f3) + def test_eq_fastpath(self): x = 1234 y = 1234 From pypy.commits at gmail.com Sun Feb 4 07:53:08 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:08 -0800 (PST) Subject: [pypy-commit] pypy default: more thorough tests for add, sub Message-ID: <5a770234.83881c0a.4fc23.67db@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93749:0c05843a6356 Date: 2018-02-04 12:51 +0100 http://bitbucket.org/pypy/pypy/changeset/0c05843a6356/ Log: more thorough tests for add, sub diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -71,10 +71,10 @@ def test_floordiv(self): for op1 in gen_signs(long_vals): + rl_op1 = rbigint.fromlong(op1) for op2 in gen_signs(long_vals): if not op2: continue - rl_op1 = rbigint.fromlong(op1) rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.floordiv(rl_op2) r2 = op1 // op2 @@ -82,10 +82,10 @@ def test_truediv(self): for op1 in gen_signs(long_vals_not_too_big): + rl_op1 = rbigint.fromlong(op1) for op2 in gen_signs(long_vals): if not op2: continue - rl_op1 = rbigint.fromlong(op1) rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.truediv(rl_op2) r2 = op1 / op2 @@ -266,36 +266,32 @@ assert rbigint._from_numberstring_parser(parser).tolong() == 1231231241 def test_add(self): - x = 123456789123456789000000L - y = 123858582373821923936744221L - for i in [-1, 1]: - for j in [-1, 1]: - f1 = rbigint.fromlong(x * i) - f2 = rbigint.fromlong(y * j) + for x in gen_signs(long_vals): + f1 = rbigint.fromlong(x) + for y in gen_signs(long_vals): + f2 = rbigint.fromlong(y) result = f1.add(f2) - assert result.tolong() == x * i + y * j + assert result.tolong() == x + y def test_int_add(self): for x in gen_signs(long_vals): + f1 = rbigint.fromlong(x) for y in signed_int_vals: - f1 = rbigint.fromlong(x) result = f1.int_add(y) assert result.tolong() == x + y def test_sub(self): - x = 12378959520302182384345L - y = 88961284756491823819191823L - for i in [-1, 1]: - for j in [-1, 1]: - f1 = rbigint.fromlong(x * i) - f2 = rbigint.fromlong(y * j) + for x in gen_signs(long_vals): + f1 = rbigint.fromlong(x) + for y in gen_signs(long_vals): + f2 = rbigint.fromlong(y) result = f1.sub(f2) - assert result.tolong() == x * i - y * j + assert result.tolong() == x - y def test_int_sub(self): - for x in gen_signs([0, 123456789123456789000000L, 1 << 100, 3 ** 10000]): + for x in gen_signs(long_vals): + f1 = rbigint.fromlong(x) for y in signed_int_vals: - f1 = rbigint.fromlong(x) result = f1.int_sub(y) assert result.tolong() == x - y @@ -316,8 +312,8 @@ def test_int_mul(self): for x in gen_signs(long_vals): + f1 = rbigint.fromlong(x) for y in signed_int_vals: - f1 = rbigint.fromlong(x) result = f1.int_mul(y) assert result.tolong() == x * y From pypy.commits at gmail.com Sun Feb 4 07:53:09 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:09 -0800 (PST) Subject: [pypy-commit] pypy default: remove unused argument (and uncovered condition) Message-ID: <5a770235.5198df0a.dcdd2.f01a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93750:cfcbece09877 Date: 2018-02-04 13:00 +0100 http://bitbucket.org/pypy/pypy/changeset/cfcbece09877/ Log: remove unused argument (and uncovered condition) diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1823,16 +1823,14 @@ ret._normalize() return ret -def _inplace_divrem1(pout, pin, n, size=0): +def _inplace_divrem1(pout, pin, n): """ Divide bigint pin by non-zero digit n, storing quotient in pout, and returning the remainder. It's OK for pin == pout on entry. """ rem = _widen_digit(0) assert n > 0 and n <= MASK - if not size: - size = pin.numdigits() - size -= 1 + size = pin.numdigits() - 1 while size >= 0: rem = (rem << SHIFT) | pin.widedigit(size) hi = rem // n From pypy.commits at gmail.com Sun Feb 4 07:53:06 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:06 -0800 (PST) Subject: [pypy-commit] pypy default: remove unreachable code (it's covered by the if a.sign == 0 just above) Message-ID: <5a770232.0db6df0a.e7815.8a26@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93748:4c9182daefc7 Date: 2018-02-04 12:30 +0100 http://bitbucket.org/pypy/pypy/changeset/4c9182daefc7/ Log: remove unreachable code (it's covered by the if a.sign == 0 just above) diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -695,9 +695,7 @@ return NULLRBIGINT if asize == 1: - if a._digits[0] == NULLDIGIT: - return NULLRBIGINT - elif a._digits[0] == ONEDIGIT: + if a._digits[0] == ONEDIGIT: return rbigint(b._digits[:b.size], a.sign * b.sign, b.size) elif bsize == 1: res = b.widedigit(0) * a.widedigit(0) From pypy.commits at gmail.com Sun Feb 4 07:53:12 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:12 -0800 (PST) Subject: [pypy-commit] pypy default: test various errors Message-ID: <5a770238.0abfdf0a.82b30.7d3a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93751:31873397de08 Date: 2018-02-04 13:10 +0100 http://bitbucket.org/pypy/pypy/changeset/31873397de08/ Log: test various errors diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -84,9 +84,10 @@ for op1 in gen_signs(long_vals_not_too_big): rl_op1 = rbigint.fromlong(op1) for op2 in gen_signs(long_vals): + rl_op2 = rbigint.fromlong(op2) if not op2: + py.test.raises(ZeroDivisionError, rl_op1.truediv, rl_op2) continue - rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.truediv(rl_op2) r2 = op1 / op2 assert r1 == r2 @@ -586,7 +587,7 @@ assert f1.tolong() == x def test_bitwise(self): - for x in gen_signs([0, 1, 5, 11, 42, 43, 3 ** 30]): + for x in gen_signs(long_vals): for y in gen_signs([0, 1, 5, 11, 42, 43, 3 ** 30, 3 ** 31]): lx = rbigint.fromlong(x) ly = rbigint.fromlong(y) @@ -679,9 +680,10 @@ def test_log(self): from rpython.rlib.rfloat import ulps_check for op in long_vals: - if not op: - continue for base in [0, 2, 4, 8, 16, 10, math.e]: + if not op: + py.test.raises(ValueError, rbigint.fromlong(op).log, base) + continue l = rbigint.fromlong(op).log(base) if base: assert ulps_check(l, math.log(op, base)) is None @@ -786,7 +788,7 @@ Rx = 1 << 130 Rx2 = 1 << 150 Ry = 1 << 127 - Ry2 = 1<< 150 + Ry2 = 1 << 150 for i in range(10): x = long(randint(Rx, Rx2)) y = long(randint(Ry, Ry2)) @@ -796,7 +798,7 @@ _div, _rem = divmod(x, y) assert div.tolong() == _div assert rem.tolong() == _rem - + def test_divmod(self): x = 12345678901234567890L for i in range(100): @@ -812,6 +814,7 @@ _div, _rem = divmod(sx, sy) assert div.tolong() == _div assert rem.tolong() == _rem + py.test.raises(ZeroDivisionError, rbigint.fromlong(x).divmod, rbigint.fromlong(0)) # testing Karatsuba stuff def test__v_iadd(self): From pypy.commits at gmail.com Sun Feb 4 07:53:14 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:14 -0800 (PST) Subject: [pypy-commit] pypy default: improve coverage of bitwise operators Message-ID: <5a77023a.93911c0a.2814f.f1d6@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93752:a52fa839d674 Date: 2018-02-04 13:50 +0100 http://bitbucket.org/pypy/pypy/changeset/a52fa839d674/ Log: improve coverage of bitwise operators diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -2548,6 +2548,8 @@ maska ^= MASK maskb ^= MASK negz = -1 + else: + assert 0, "unreachable" # JRH: The original logic here was to allocate the result value (z) # as the longer of the two operands. However, there are some cases diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -588,8 +588,8 @@ def test_bitwise(self): for x in gen_signs(long_vals): - for y in gen_signs([0, 1, 5, 11, 42, 43, 3 ** 30, 3 ** 31]): - lx = rbigint.fromlong(x) + lx = rbigint.fromlong(x) + for y in gen_signs(long_vals): ly = rbigint.fromlong(y) for mod in "xor and_ or_".split(): res1 = getattr(lx, mod)(ly).tolong() @@ -597,11 +597,9 @@ assert res1 == res2 def test_int_bitwise(self): - for x in gen_signs([0, 1, 5, 11, 42, 43, 2 ** 30]): - for y in gen_signs([0, 1, 5, 11, 42, 43, 3 ** 30, 2 ** 31]): - if y != intmask(y): - continue # skip 'y' too large for 32-bit - lx = rbigint.fromlong(x) + for x in gen_signs(long_vals): + lx = rbigint.fromlong(x) + for y in signed_int_vals: for mod in "xor and_ or_".split(): res1 = getattr(lx, 'int_' + mod)(y).tolong() res2 = getattr(operator, mod)(x, y) From pypy.commits at gmail.com Sun Feb 4 07:53:16 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:16 -0800 (PST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5a77023c.131f1c0a.712ce.bb77@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93753:03e7d032c07f Date: 2018-02-04 13:52 +0100 http://bitbucket.org/pypy/pypy/changeset/03e7d032c07f/ Log: merge heads diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -23,3 +23,16 @@ added, then the performance using mapdict is linear in the number of attributes. This is now fixed (by switching to a regular dict after 80 attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,6 +208,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,10 +3,20 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -157,6 +167,25 @@ assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + def test_shortcut(self): + # test that instances of classes that are defined in C become an + # instance of W_BaseCPyObject and thus can be converted faster back to + # their pyobj, because they store a pointer to it directly. + if self.runappdirect: + skip("can't run with -A") + module = self.import_module(name='foo') + obj = module.fooType() + assert self._check_uses_shortcut(obj) + # W_TypeObjects use shortcut + assert self._check_uses_shortcut(object) + assert self._check_uses_shortcut(type) + # None, True, False use shortcut + assert self._check_uses_shortcut(None) + assert self._check_uses_shortcut(True) + assert self._check_uses_shortcut(False) + assert not self._check_uses_shortcut(1) + assert not self._check_uses_shortcut(object()) + def test_multiple_inheritance1(self): module = self.import_module(name='foo') obj = module.UnicodeSubtype(u'xyz') From pypy.commits at gmail.com Sun Feb 4 07:53:04 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:53:04 -0800 (PST) Subject: [pypy-commit] pypy default: be a lot more systematic about testing the rbigint.int_* variants. This Message-ID: <5a770230.0dc4df0a.cb38b.a000@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93747:f225421c6b26 Date: 2018-02-03 23:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f225421c6b26/ Log: be a lot more systematic about testing the rbigint.int_* variants. This increases test coverage diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -17,15 +17,31 @@ from hypothesis import given, strategies +def gen_signs(l): + for s in l: + if s == 0: + yield s + else: + yield s + yield -s + long_vals_not_too_big = range(17) + [ - 37, 50, + 37, 39, 50, 127, 128, 129, 511, 512, 513, sys.maxint, sys.maxint + 1, + 12345678901234567890L, 123456789123456789000000L, - ] +] long_vals = long_vals_not_too_big + [ 1 << 100, 3 ** 10000] +int_vals = range(33) + [ + 1000, + 0x11111111, 0x11111112, 8888, + 9999, sys.maxint, 2 ** 19, 2 ** 18 - 1 +] +signed_int_vals = list(gen_signs(int_vals)) + [-sys.maxint-1] + class TestRLong(object): def test_simple(self): for op1 in [-2, -1, 0, 1, 2, 50]: @@ -121,12 +137,14 @@ rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.mod(rl_op2) r2 = op1 % op2 - print op1, op2 + assert r1.tolong() == r2 def test_int_mod(self): for x in gen_signs(long_vals): - for y in gen_signs([1, 2, 4, 8, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: + if not y: + continue op1 = rbigint.fromlong(x) r1 = op1.int_mod(y) r2 = x % y @@ -158,13 +176,6 @@ assert not (a1 == a3) -def gen_signs(l): - for s in l: - if s == 0: - yield s - else: - yield s - yield -s def bigint(lst, sign): for digit in lst: @@ -266,7 +277,7 @@ def test_int_add(self): for x in gen_signs(long_vals): - for y in gen_signs([0, 1, 9999, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_add(y) assert result.tolong() == x + y @@ -283,7 +294,7 @@ def test_int_sub(self): for x in gen_signs([0, 123456789123456789000000L, 1 << 100, 3 ** 10000]): - for y in gen_signs([0, 1, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_sub(y) assert result.tolong() == x - y @@ -304,8 +315,8 @@ assert result.tolong() == x * x def test_int_mul(self): - for x in gen_signs([39, 128, 111111111, 123456789123456789000000L, 1 << 100, 3 ** 10000]): - for y in gen_signs([0, 1, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for x in gen_signs(long_vals): + for y in signed_int_vals: f1 = rbigint.fromlong(x) result = f1.int_mul(y) assert result.tolong() == x * y @@ -394,14 +405,14 @@ def test_int_comparison(self): for x in gen_signs(long_vals): - for y in gen_signs([0, 1, 0x11111111, 0x11111112, 8888, sys.maxint, 2 ** 19, 2 ** 18 - 1]): + for y in signed_int_vals: f1 = rbigint.fromlong(x) - assert (x < y) == f1.int_lt(y) - assert (x <= y) == f1.int_le(y) - assert (x > y) == f1.int_gt(y) - assert (x >= y) == f1.int_ge(y) - assert (x == y) == f1.int_eq(y) - assert (x != y) == f1.int_ne(y) + assert (x < y) == f1.int_lt(y) + assert (x <= y) == f1.int_le(y) + assert (x > y) == f1.int_gt(y) + assert (x >= y) == f1.int_ge(y) + assert (x == y) == f1.int_eq(y) + assert (x != y) == f1.int_ne(y) def test_order(self): f6 = rbigint.fromint(6) From pypy.commits at gmail.com Sun Feb 4 07:57:03 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 04 Feb 2018 04:57:03 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge default Message-ID: <5a77031f.42a6df0a.ed2ab.7f9b@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93754:4f4cee77f1c2 Date: 2018-02-04 13:56 +0100 http://bitbucket.org/pypy/pypy/changeset/4f4cee77f1c2/ Log: merge default diff too long, truncating to 2000 out of 92474 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -44,3 +44,10 @@ d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,2 +1,3 @@ pytest hypothesis +vmprof diff --git a/extra_tests/test_json.py b/extra_tests/test_json.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_json.py @@ -0,0 +1,33 @@ +import pytest +import json +from hypothesis import given, strategies + +def is_(x, y): + return type(x) is type(y) and x == y + +def test_no_ensure_ascii(): + assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') + assert is_(json.dumps("\xc0", ensure_ascii=False), '"\xc0"') + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps((u"\u1234", "\xc0"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps(("\xc0", u"\u1234"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + +def test_issue2191(): + assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_textio.py b/extra_tests/test_textio.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_textio.py @@ -0,0 +1,48 @@ +from hypothesis import given, strategies as st + +from io import BytesIO, TextIOWrapper +import os + +def translate_newlines(text): + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + return text.replace('\n', os.linesep) + + at st.composite +def st_readline_universal( + draw, st_nlines=st.integers(min_value=0, max_value=10)): + n_lines = draw(st_nlines) + lines = draw(st.lists( + st.text(st.characters(blacklist_characters='\r\n')), + min_size=n_lines, max_size=n_lines)) + limits = [] + for line in lines: + limit = draw(st.integers(min_value=0, max_value=len(line) + 5)) + limits.append(limit) + limits.append(-1) + endings = draw(st.lists( + st.sampled_from(['\n', '\r', '\r\n']), + min_size=n_lines, max_size=n_lines)) + return ( + ''.join(line + ending for line, ending in zip(lines, endings)), + limits) + + at given(data=st_readline_universal(), + mode=st.sampled_from(['\r', '\n', '\r\n', '', None])) +def test_readline(data, mode): + txt, limits = data + textio = TextIOWrapper( + BytesIO(txt.encode('utf-8', 'surrogatepass')), + encoding='utf-8', errors='surrogatepass', newline=mode) + lines = [] + for limit in limits: + line = textio.readline(limit) + if limit >= 0: + assert len(line) <= limit + if line: + lines.append(line) + elif limit: + break + if mode is None: + txt = translate_newlines(txt) + assert txt.startswith(u''.join(lines)) diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -40,6 +40,10 @@ import linecache from operator import attrgetter from collections import namedtuple +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # These constants are from Include/code.h. CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8 @@ -230,7 +234,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -286,7 +286,7 @@ self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) u.close() - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -43,11 +43,12 @@ unicodetype = unicode except NameError: unicodetype = () + template = "%s: %s: %s\n" try: message = str(message) except UnicodeEncodeError: - pass - s = "%s: %s: %s\n" % (lineno, category.__name__, message) + template = unicode(template) + s = template % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,7 +12,8 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, int): + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,6 +7,16 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -17,10 +17,13 @@ """ from __future__ import division -import time as _time +import time as _timemodule import math as _math import struct as _struct +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + _SENTINEL = object() def _cmp(x, y): @@ -179,7 +182,7 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. @@ -247,7 +250,7 @@ else: push(ch) newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) + return _timemodule.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): @@ -433,7 +436,7 @@ raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) -class timedelta(object): +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -489,7 +492,7 @@ if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -667,7 +670,7 @@ timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) -class date(object): +class date(dateinterop): """Concrete date type. Constructors: @@ -707,12 +710,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -724,13 +727,13 @@ @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t) @classmethod @@ -1061,7 +1064,7 @@ _tzinfo_class = tzinfo -class time(object): +class time(timeinterop): """Time with time zone. Constructors: @@ -1097,14 +1100,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1408,15 +1411,20 @@ if isinstance(year, bytes) and len(year) == 10 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1461,7 +1469,7 @@ A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - converter = _time.localtime if tz is None else _time.gmtime + converter = _timemodule.localtime if tz is None else _timemodule.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) @@ -1470,7 +1478,7 @@ @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - return cls._from_timestamp(_time.gmtime, t, None) + return cls._from_timestamp(_timemodule.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): @@ -1488,18 +1496,18 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." - t = _time.time() + t = _timemodule.time() return cls.utcfromtimestamp(t) @classmethod @@ -1797,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1805,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -20,6 +20,7 @@ or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" __metaclass__ = _structseq.structseqtype + name = "resource.struct_rusage" ru_utime = _structseq.structseqfield(0, "user time used") ru_stime = _structseq.structseqfield(1, "system time used") diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -149,7 +149,7 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X:: +On Mac OS X: Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -355,7 +355,11 @@ containers (as list items or in sets for example), the exact rule of equality used is "``if x is y or x == y``" (on both CPython and PyPy); as a consequence, because all ``nans`` are identical in PyPy, you -cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__). +Another consequence is that ``cmp(float('nan'), float('nan')) == 0``, because +``cmp`` checks with ``is`` first whether the arguments are identical (there is +no good value to return from this call to ``cmp``, because ``cmp`` pretends +that there is a total order on floats, but that is wrong for NaNs). .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of @@ -541,6 +545,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,22 +1,38 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:d56dadcef996 - -.. branch: cppyy-packaging -Cleanup and improve cppyy packaging - -.. branch: docs-osx-brew-openssl - -.. branch: keep-debug-symbols -Add a smartstrip tool, which can optionally keep the debug symbols in a -separate file, instead of just stripping them away. Use it in packaging - -.. branch: bsd-patches -Fix failures on FreeBSD, contributed by David Naylor as patches on the issue -tracker (issues 2694, 2695, 2696, 2697) - -.. branch: run-extra-tests -Run extra_tests/ in buildbot +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -0,0 +1,42 @@ +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst --- a/pypy/doc/whatsnew-pypy2-5.6.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst @@ -101,7 +101,7 @@ .. branch: newinitwarn -Match CPython's stricter handling of __new/init__ arguments +Match CPython's stricter handling of ``__new__``/``__init__`` arguments .. branch: openssl-1.1 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -11,7 +11,7 @@ To build pypy-c you need a working python environment, and a C compiler. It is possible to translate with a CPython 2.6 or later, but this is not -the preferred way, because it will take a lot longer to run � depending +the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -25,8 +25,10 @@ This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link -was checked in Nov 2016). Note that the compiler suite will be installed in -``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``. +was checked in Nov 2016). Note that the compiler suite may be installed in +``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python`` +or in +``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``. A current version of ``setuptools`` will be able to find it there. For Windows 10, you must right-click the download, and under ``Properties`` -> ``Compatibility`` mark it as ``Run run this program in comatibility mode for`` @@ -41,7 +43,6 @@ ----------------------------------- We routinely test translation using v9, also known as Visual Studio 2008. -Our buildbot is still using the Express Edition, not the compiler noted above. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -81,6 +82,31 @@ .. _build instructions: http://pypy.org/download.html#building-from-source +Setting Up Visual Studio for building SSL in Python3 +---------------------------------------------------- + +On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after +translation. However ``distutils`` does not support the Micorosft-provided Visual C +compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The +traditional solution to this problem is to install the ``setuptools`` module +via running ``-m ensurepip`` which installs ``pip`` and ``setuptools``. However +``pip`` requires ``ssl``. So we have a chicken-and-egg problem: ``ssl`` depends on +``cffi`` which depends on ``setuptools``, which depends on ``ensurepip``, which +depends on ``ssl``. + +In order to solve this, the buildbot sets an environment varaible that helps +``distutils`` find the compiler without ``setuptools``:: + + set VS90COMNTOOLS=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin + +or whatever is appropriate for your machine. Note that this is not enough, you +must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the +``...\9.0\VC`` directory, and edit it, changing the lines that set +``VCINSTALLDIR`` and ``WindowsSdkDir``:: + + set VCINSTALLDIR=%~dp0\ + set WindowsSdkDir=%~dp0\..\WinSDK\ + Preparing Windows for the large build ------------------------------------- diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import we_are_translated from pypy.interpreter.astcompiler import ast, misc, symtable @@ -266,7 +266,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -276,9 +276,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1096,17 +1096,21 @@ encoding = self.compile_info.encoding flags = self.compile_info.flags unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - try: - sub_strings_w = [parsestring.parsestr(space, encoding, atom_node.get_child(i).get_value(), - unicode_literals) - for i in range(atom_node.num_children())] - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise self.error('(unicode error) %s' % errmsg, atom_node) + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), + unicode_literals)) + except error.OperationError as e: + if not e.match(space, space.w_UnicodeError): + raise + # UnicodeError in literal: turn into SyntaxError + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + if child is None: + child = atom_node + raise self.error('(unicode error) %s' % errmsg, child) # This implements implicit string concatenation. if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1246,3 +1246,15 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-1: truncated \\xXX escape") + input = "u'\\x1'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,6 +208,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -85,13 +85,17 @@ # permissive parsing of the given list of tokens; it relies on # the real parsing done afterwards to give errors. it.skip_newlines() - it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") - if it.skip(pygram.tokens.STRING): - it.skip_newlines() - while (it.skip_name("from") and + docstring_possible = True + while True: + it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") + if docstring_possible and it.skip(pygram.tokens.STRING): + it.skip_newlines() + docstring_possible = False + if not (it.skip_name("from") and it.skip_name("__future__") and it.skip_name("import")): + break it.skip(pygram.tokens.LPAR) # optionally # return in 'last_position' any line-column pair that points # somewhere inside the last __future__ import statement diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py --- a/pypy/interpreter/pyparser/test/test_future.py +++ b/pypy/interpreter/pyparser/test/test_future.py @@ -208,3 +208,13 @@ 'from __future__ import with_statement;') f = run(s, (2, 23)) assert f == fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_WITH_STATEMENT + +def test_future_doc_future(): + # for some reason people do this :-[ + s = ''' +from __future__ import generators +"Docstring" +from __future__ import division + ''' + f = run(s, (4, 24)) + assert f == fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,4 +1,7 @@ -from pypy.interpreter.unicodehelper import encode_utf8, decode_utf8 +import pytest +import struct +from pypy.interpreter.unicodehelper import ( + encode_utf8, decode_utf8, unicode_encode_utf_32_be) class FakeSpace: pass @@ -24,3 +27,23 @@ assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] + + at pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) +def test_utf32_surrogates(unich): + assert (unicode_encode_utf_32_be(unich, 1, None) == + struct.pack('>i', ord(unich))) + with pytest.raises(UnicodeEncodeError): + unicode_encode_utf_32_be(unich, 1, None, allow_surrogates=False) + + def replace_with(ru, rs): + def errorhandler(errors, enc, msg, u, startingpos, endingpos): + if errors == 'strict': + raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg) + return ru, rs, endingpos + return unicode_encode_utf_32_be( + u"<%s>" % unich, 3, None, + errorhandler, allow_surrogates=False) + + assert replace_with(u'rep', None) == u''.encode('utf-32-be') + assert (replace_with(None, '\xca\xfe\xca\xfe') == + '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,7 +1,11 @@ +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder +from rpython.rlib import runicode +from rpython.rlib.runicode import ( + default_unicode_error_encode, default_unicode_error_decode, + MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR) from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import specialize -from rpython.rlib import runicode -from pypy.module._codecs import interp_codecs @specialize.memo() def decode_error_handler(space): @@ -37,6 +41,7 @@ # These functions take and return unwrapped rpython strings and unicodes def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result, consumed = runicode.str_decode_unicode_escape( @@ -71,3 +76,229 @@ uni, len(uni), "strict", errorhandler=None, allow_surrogates=True) + +# ____________________________________________________________ +# utf-32 + +def str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native") + return result, length + +def str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big") + return result, length + +def str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little") + return result, length + +def py3k_str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2) + return result, length + +def py3k_str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be') + return result, length + +def py3k_str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le') + return result, length + +BOM32_DIRECT = intmask(0x0000FEFF) +BOM32_REVERSE = intmask(0xFFFE0000) + +def str_decode_utf_32_helper(s, size, errors, final=True, + errorhandler=None, + byteorder="native", + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_decode + bo = 0 + + if BYTEORDER == 'little': + iorder = [0, 1, 2, 3] + else: + iorder = [3, 2, 1, 0] + + # Check for BOM marks (U+FEFF) in the input and adjust current + # byte order setting accordingly. In native mode, the leading BOM + # mark is skipped, in all other modes, it is copied to the output + # stream as-is (giving a ZWNBSP character). + pos = 0 + if byteorder == 'native': + if size >= 4: + bom = intmask( + (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | + (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) + if BYTEORDER == 'little': + if bom == BOM32_DIRECT: + pos += 4 + bo = -1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = 1 + else: + if bom == BOM32_DIRECT: + pos += 4 + bo = 1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = -1 + elif byteorder == 'little': + bo = -1 + else: + bo = 1 + if size == 0: + return u'', 0, bo + if bo == -1: + # force little endian + iorder = [0, 1, 2, 3] + elif bo == 1: + # force big endian + iorder = [3, 2, 1, 0] + + result = UnicodeBuilder(size // 4) + + while pos < size: + # remaining bytes at the end? (size should be divisible by 4) + if len(s) - pos < 4: + if not final: + break + r, pos = errorhandler(errors, public_encoding_name, + "truncated data", + s, pos, len(s)) + result.append(r) + if len(s) - pos < 4: + break + continue + ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) | + (ord(s[pos + iorder[1]]) << 8) | ord(s[pos + iorder[0]])) + if ch >= 0x110000: + r, pos = errorhandler(errors, public_encoding_name, + "codepoint not in range(0x110000)", + s, pos, len(s)) + result.append(r) + continue + + if MAXUNICODE < 65536 and ch >= 0x10000: + ch -= 0x10000L + result.append(unichr(0xD800 + (ch >> 10))) + result.append(unichr(0xDC00 + (ch & 0x03FF))) + else: + result.append(UNICHR(ch)) + pos += 4 + return result.build(), pos, bo + +def _STORECHAR32(result, CH, byteorder): + c0 = chr(((CH) >> 24) & 0xff) + c1 = chr(((CH) >> 16) & 0xff) + c2 = chr(((CH) >> 8) & 0xff) + c3 = chr((CH) & 0xff) + if byteorder == 'little': + result.append(c3) + result.append(c2) + result.append(c1) + result.append(c0) + else: + result.append(c0) + result.append(c1) + result.append(c2) + result.append(c3) + +def unicode_encode_utf_32_helper(s, size, errors, + errorhandler=None, + allow_surrogates=True, + byteorder='little', + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_encode + if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() + return "" + + result = StringBuilder(size * 4 + 4) + if byteorder == 'native': + _STORECHAR32(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 + ch2 = 0 + if not allow_surrogates and 0xD800 <= ch < 0xE000: + ru, rs, pos = errorhandler( + errors, public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + if rs is not None: + # py3k only + if len(rs) % 4 != 0: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + result.append(rs) + continue + for ch in ru: + if ord(ch) < 0xD800: + _STORECHAR32(result, ord(ch), byteorder) + else: + errorhandler( + 'strict', public_encoding_name, + 'surrogates not allowed', s, pos - 1, pos) + continue + if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size: + ch2 = ord(s[pos]) + if 0xDC00 <= ch2 < 0xE000: + ch = (((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000 + pos += 1 + _STORECHAR32(result, ch, byteorder) + + return result.build() + +def unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native") + +def unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big") + +def unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little") + +def py3k_unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native", + 'utf-32-' + BYTEORDER2) + +def py3k_unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big", + 'utf-32-be') + +def py3k_unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little", + 'utf-32-le') diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -2,11 +2,13 @@ Interp-level implementation of the basic space operations. """ +import math + from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from rpython.rlib.runicode import UNICHR -from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away +from rpython.rlib.rfloat import isfinite, round_double, round_away from rpython.rlib import rfloat import __builtin__ @@ -151,7 +153,7 @@ else: # finite x, and ndigits is not unreasonably large z = round_double(number, ndigits) - if isinf(z): + if math.isinf(z): raise oefmt(space.w_OverflowError, "rounded value too large to represent") return space.newfloat(z) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -404,6 +404,7 @@ def test_cmp(self): + assert cmp(float('nan'), float('nan')) == 0 assert cmp(9,9) == 0 assert cmp(0,9) < 0 assert cmp(9,0) > 0 diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -58,6 +58,14 @@ } +class PyPyDateTime(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + } + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -108,6 +116,7 @@ "thread": ThreadModule, "intop": IntOpModule, "os": OsModule, + '_pypydatetime': PyPyDateTime, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -0,0 +1,24 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from rpython.tool.sourcetools import func_with_new_name + +def create_class(name): + class W_Class(W_Root): + 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + def descr_new__(space, w_type): + return space.allocate_instance(W_Class, w_type) + + W_Class.typedef = TypeDef(name, + __new__ = interp2app(func_with_new_name( + W_Class.descr_new__.im_func, + '%s_new' % (name,))), + ) + W_Class.typedef.acceptable_as_base_class = True + return W_Class + +W_DateTime_Time = create_class('pypydatetime_time') +W_DateTime_Date = create_class('pypydatetime_date') +W_DateTime_Delta = create_class('pypydatetime_delta') + + diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, + w_extra_compile_args=None): try: import cffi from cffi import FFI # <== the system one, which @@ -55,10 +56,14 @@ sources = [] if w_extra_source is not None: sources.append(space.str_w(w_extra_source)) + kwargs = {} + if w_extra_compile_args is not None: + kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args) ext = ffiplatform.get_extension(c_file, module_name, include_dirs=[str(rdir)], export_symbols=['_cffi_pypyinit_' + base_module_name], - sources=sources) + sources=sources, + **kwargs) ffiplatform.compile(str(rdir), ext) for extension in ['so', 'pyd', 'dylib']: @@ -2054,3 +2059,51 @@ "Such structs are only supported as return value if the function is " "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()" "+ffibuilder.set_source() and not taking a final '...' argument)") + + def test_gcc_visibility_hidden(self): + import sys + if sys.platform == 'win32': + skip("test for gcc/clang") + ffi, lib = self.prepare(""" + int f(int); + """, "test_gcc_visibility_hidden", """ + int f(int a) { return a + 40; } + """, extra_compile_args=['-fvisibility=hidden']) + assert lib.f(2) == 42 + + def test_override_default_definition(self): + ffi, lib = self.prepare(""" + typedef long int16_t, char16_t; + """, "test_override_default_definition", """ + """) + assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long") + + def test_char16_char32_plain_c(self): + ffi, lib = self.prepare(""" + char16_t foo_2bytes(char16_t); + char32_t foo_4bytes(char32_t); + """, "test_char16_char32_type_nocpp", """ + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) + typedef uint_least16_t char16_t; + typedef uint_least32_t char32_t; + #endif + + char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); } + char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); } + """, min_version=(1, 11, 0)) + assert lib.foo_2bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\U00012345') == u'\U0001236f' + raises(TypeError, lib.foo_2bytes, u'\U00012345') + raises(TypeError, lib.foo_2bytes, 1234) + raises(TypeError, lib.foo_4bytes, 1234) + + def test_loader_spec(self): + import sys + ffi, lib = self.prepare("", "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1,10 +1,12 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import we_are_translated, not_rpython from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib import runicode from rpython.rlib.runicode import code_to_unichr, MAXUNICODE from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter import unicodehelper class VersionTag(object): @@ -210,7 +212,8 @@ def xmlcharrefreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -236,7 +239,8 @@ def backslashreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -363,19 +367,23 @@ raise oefmt(space.w_TypeError, "handler must be callable") # ____________________________________________________________ -# delegation to runicode +# delegation to runicode/unicodehelper -from rpython.rlib import runicode +def _find_implementation(impl_name): + try: + func = getattr(unicodehelper, impl_name) + except AttributeError: + func = getattr(runicode, impl_name) + return func def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(uni=unicode, errors='text_or_none') def wrap_encoder(space, uni, errors="strict"): if errors is None: errors = 'strict' state = space.fromcache(CodecState) - func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) return space.newtuple([space.newbytes(result), space.newint(len(uni))]) wrap_encoder.func_name = rname @@ -383,7 +391,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(string='bufferstr', errors='text_or_none', w_final=WrappedDefault(False)) def wrap_decoder(space, string, errors="strict", w_final=None): @@ -391,7 +399,6 @@ errors = 'strict' final = space.is_true(w_final) state = space.fromcache(CodecState) - func = getattr(runicode, rname) result, consumed = func(string, len(string), errors, final, state.decode_error_handler) return space.newtuple([space.newunicode(result), space.newint(consumed)]) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -115,10 +115,10 @@ raises(TypeError, charmap_decode, '\xff', "strict", {0xff: 0x110000}) assert (charmap_decode("\x00\x01\x02", "strict", {0: 0x10FFFF, 1: ord('b'), 2: ord('c')}) == - u"\U0010FFFFbc", 3) + (u"\U0010FFFFbc", 3)) assert (charmap_decode("\x00\x01\x02", "strict", {0: u'\U0010FFFF', 1: u'b', 2: u'c'}) == - u"\U0010FFFFbc", 3) + (u"\U0010FFFFbc", 3)) def test_escape_decode_errors(self): from _codecs import escape_decode as decode @@ -537,8 +537,12 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' - def test_backslahreplace(self): - assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_backslashreplace(self): + sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" + expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + assert sin.encode('ascii', 'backslashreplace') == expected + expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff" + assert sin.encode("iso-8859-15", "backslashreplace") == expected From pypy.commits at gmail.com Mon Feb 5 04:56:19 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 05 Feb 2018 01:56:19 -0800 (PST) Subject: [pypy-commit] pypy vmprof-resolve_addr: don't use a hardcoded address, but use a more proper way of discovering one Message-ID: <5a782a43.85631c0a.ab1cd.461b@mx.google.com> Author: Antonio Cuni Branch: vmprof-resolve_addr Changeset: r93755:0bf081e65685 Date: 2018-02-05 10:05 +0100 http://bitbucket.org/pypy/pypy/changeset/0bf081e65685/ Log: don't use a hardcoded address, but use a more proper way of discovering one diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -1,6 +1,8 @@ import py import sys from rpython.tool.udir import udir +from rpython.rlib import rvmprof +from rpython.rtyper.lltypesystem import rffi, lltype from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): @@ -11,6 +13,9 @@ cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2'))) cls.w_plain = cls.space.wrap(not cls.runappdirect and '__pypy__' not in sys.builtin_module_names) + # + addr = rffi.cast(lltype.Signed, rvmprof.get_address_of_vmprof_start_sampling()) + cls.w_addr_of_vmprof_start_sampling = cls.space.wrap(addr) def test_import_vmprof(self): tmpfile = open(self.tmpfilename, 'wb') @@ -152,8 +157,7 @@ def test_resolve_addr(self): import _vmprof - addr = 0x517450 # XXX dont' hardcode + addr = self.addr_of_vmprof_start_sampling name, lineno, srcfile = _vmprof.resolve_addr(addr) - assert name == 'PyLong_AsLong' - assert lineno == 0 - assert srcfile == 'python' + assert name == 'vmprof_start_sampling' + assert srcfile.endswith('rvmprof.c') diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -84,3 +84,15 @@ vmprof_tl_stack.setraw(x) start_sampling() + +# -------------- +# used by tests +# -------------- + +def get_address_of_vmprof_start_sampling(): + """ + This is used only for testing that vmp_resolve_addr can find the correct + symbol name + """ + fn = _get_vmprof().cintf.vmprof_start_sampling + return rffi.cast(rffi.VOIDP, fn) diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -204,8 +204,7 @@ class TestSymbolTable(object): def test_vmp_resolve_addr(self): - # XXX: don't hardcode this addr - addr = rffi.cast(rffi.VOIDP, 0x517450) + addr = rvmprof.get_address_of_vmprof_start_sampling() name, lineno, srcfile = rvmprof.resolve_addr(addr) - assert name == 'PyLong_AsLong' - + assert name == 'vmprof_start_sampling' + assert srcfile.endswith('rvmprof.c') From pypy.commits at gmail.com Mon Feb 5 04:56:21 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 05 Feb 2018 01:56:21 -0800 (PST) Subject: [pypy-commit] pypy vmprof-resolve_addr: initialize the pointers in the very same way as the C version does; this is apparently part of the API of vmp_resolve_address Message-ID: <5a782a45.ba8adf0a.1921d.fbb1@mx.google.com> Author: Antonio Cuni Branch: vmprof-resolve_addr Changeset: r93756:451e009ea2ec Date: 2018-02-05 10:55 +0100 http://bitbucket.org/pypy/pypy/changeset/451e009ea2ec/ Log: initialize the pointers in the very same way as the C version does; this is apparently part of the API of vmp_resolve_address diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -192,11 +192,10 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, name_len) as name_p: with lltype.scoped_alloc(rffi.CCHARP.TO, srcfile_len) as srcfile_p: with lltype.scoped_alloc(rffi.INTP.TO, 1) as lineno_p: - # XXX vmp_resolve_addr checks whether the first char is 0 - # before calling dladdr, not sure why. Investigate before - # merging. name_p[0] = '\0' - srcfile_p[0] = '\0' + srcfile_p[0] = '-' + srcfile_p[1] = '\0' + lineno_p[0] = rffi.cast(rffi.INT, 0) res = self.cintf.vmp_resolve_addr(addr, name_p, name_len, lineno_p, srcfile_p, srcfile_len) if res != 0: From pypy.commits at gmail.com Mon Feb 5 06:03:30 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 03:03:30 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: fix rbigint.int_divmod Message-ID: <5a783a02.0ee51c0a.1af56.6c10@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93757:744cb1dc4f2e Date: 2018-02-05 11:49 +0100 http://bitbucket.org/pypy/pypy/changeset/744cb1dc4f2e/ Log: fix rbigint.int_divmod diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -949,24 +949,23 @@ if not int_in_valid_range(w) or (wsign == -1 and v.sign != wsign): # Just fallback. return v.divmod(rbigint.fromint(w)) - + digit = abs(w) assert digit > 0 div, mod = _divrem1(v, digit) div.sign = v.sign * wsign - - if v.sign != wsign: + if v.sign < 0: + mod = -mod + if mod and v.sign * wsign == -1: + mod += w if div.sign == 0: div = ONENEGATIVERBIGINT else: div = div.int_sub(1) - mod = w - mod - - mod = rbigint.fromint(wsign * mod) - + mod = rbigint.fromint(mod) return div, mod - + @jit.elidable def pow(a, b, c=None): negativeOutput = False # if x<0 return negative output diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -902,8 +902,6 @@ for sx, sy in (1, 1), (1, -1), (-1, -1), (-1, 1): sx *= x sy *= y - if sx == 0 and sy == 1: - import pdb; pdb.set_trace() if sy == sys.maxint + 1: continue f1 = rbigint.fromlong(sx) From pypy.commits at gmail.com Mon Feb 5 06:03:32 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 03:03:32 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: remove copy-pasted code between mod and int_mod Message-ID: <5a783a04.46821c0a.f9f74.29ac@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93758:f6e9839f73ac Date: 2018-02-05 12:01 +0100 http://bitbucket.org/pypy/pypy/changeset/f6e9839f73ac/ Log: remove copy-pasted code between mod and int_mod diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -837,32 +837,9 @@ return NULLRBIGINT if other.sign != 0 and other.numdigits() == 1: - digit = other.digit(0) - if digit == 1: - return NULLRBIGINT - elif digit == 2: - modm = self.digit(0) & 1 - if modm: - return ONENEGATIVERBIGINT if other.sign == -1 else ONERBIGINT - return NULLRBIGINT - elif digit & (digit - 1) == 0: - mod = self.int_and_(digit - 1) - else: - # Perform - size = UDIGIT_TYPE(self.numdigits() - 1) - - if size > 0: - wrem = self.widedigit(size) - while size > 0: - size -= 1 - wrem = ((wrem << SHIFT) | self.digit(size)) % digit - rem = _store_digit(wrem) - else: - rem = _store_digit(self.digit(0) % digit) - - if rem == 0: - return NULLRBIGINT - mod = rbigint([rem], -1 if self.sign < 0 else 1, 1) + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_mod(otherint) else: div, mod = _divrem(self, other) if mod.sign * other.sign == -1: @@ -892,7 +869,7 @@ else: # Perform size = UDIGIT_TYPE(self.numdigits() - 1) - + if size > 0: wrem = self.widedigit(size) while size > 0: From pypy.commits at gmail.com Mon Feb 5 06:38:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 03:38:20 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: remove code duplication in floordiv Message-ID: <5a78422c.8fb1df0a.f7c76.f86e@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93759:5c8a03028ed8 Date: 2018-02-05 12:27 +0100 http://bitbucket.org/pypy/pypy/changeset/5c8a03028ed8/ Log: remove code duplication in floordiv diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -783,12 +783,10 @@ @jit.elidable def floordiv(self, other): - if self.sign == 1 and other.numdigits() == 1 and other.sign == 1: - digit = other.digit(0) - if digit == 1: - return self - elif digit & (digit - 1) == 0: - return self.rqshift(ptwotable[digit]) + if other.numdigits() == 1: + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_floordiv(otherint) div, mod = _divrem(self, other) if mod.sign * other.sign == -1: @@ -800,13 +798,13 @@ def div(self, other): return self.floordiv(other) - + @jit.elidable def int_floordiv(self, b): if not int_in_valid_range(b): # Fallback to long. return self.floordiv(rbigint.fromint(b)) - + if b == 0: raise ZeroDivisionError("long division by zero") @@ -818,7 +816,7 @@ return self elif digit & (digit - 1) == 0: return self.rqshift(ptwotable[digit]) - + div, mod = _divrem1(self, digit) if mod != 0 and self.sign * (-1 if b < 0 else 1) == -1: @@ -830,7 +828,7 @@ def int_div(self, other): return self.int_floordiv(other) - + @jit.elidable def mod(self, other): if self.sign == 0: From pypy.commits at gmail.com Mon Feb 5 06:38:22 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 03:38:22 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: clean up whitespace Message-ID: <5a78422e.c7a9df0a.1de89.23b0@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93760:ef798e871400 Date: 2018-02-05 12:37 +0100 http://bitbucket.org/pypy/pypy/changeset/ef798e871400/ Log: clean up whitespace diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -41,7 +41,7 @@ UNSIGNED_TYPE = lltype.Unsigned LONG_TYPE = rffi.LONGLONG ULONG_TYPE = rffi.ULONGLONG - + MASK = int((1 << SHIFT) - 1) FLOAT_MULTIPLIER = float(1 << SHIFT) @@ -100,7 +100,7 @@ def _unsigned_widen_digit(x): return rffi.cast(ULONG_TYPE, x) - + @specialize.argtype(0) def _store_digit(x): return rffi.cast(STORE_TYPE, x) @@ -143,16 +143,16 @@ """This is a reimplementation of longs using a list of digits.""" _immutable_ = True _immutable_fields_ = ["_digits[*]", "size", "sign"] - + def __init__(self, digits=NULLDIGITS, sign=0, size=0): if not we_are_translated(): _check_digits(digits) make_sure_not_resized(digits) self._digits = digits - + assert size >= 0 self.size = size or len(digits) - + self.sign = sign # __eq__ and __ne__ method exist for testingl only, they are not RPython! @@ -183,7 +183,7 @@ to have enough room to contain two digits.""" return _unsigned_widen_digit(self._digits[x]) uwidedigit._always_inline_ = True - + def udigit(self, x): """Return the x'th digit, as an unsigned int.""" return _load_unsigned_digit(self._digits[x]) @@ -220,7 +220,7 @@ else: return NULLRBIGINT - + if carry: return rbigint([_store_digit(ival & MASK), _store_digit(carry)], sign, 2) @@ -530,7 +530,7 @@ def int_eq(self, other): """ eq with int """ if not int_in_valid_range(other): - # Fallback to Long. + # Fallback to Long. return self.eq(rbigint.fromint(other)) if self.numdigits() > 1: @@ -592,14 +592,14 @@ osign = 0 elif other < 0: osign = -1 - + if self.sign > osign: return False elif self.sign < osign: return True digits = self.numdigits() - + if digits > 1: if osign == 1: return False @@ -753,7 +753,7 @@ asize = self.numdigits() digit = abs(b) - + bsign = -1 if b < 0 else 1 if digit == 1: @@ -1026,7 +1026,7 @@ if bi & j: z = _help_mult(z, a, c) j >>= 1 - + else: # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82) @@ -1151,7 +1151,7 @@ # Left-to-right binary exponentiation (HAC Algorithm 14.79) # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf j = 1 << (SHIFT-1) - + while j != 0: z = _help_mult(z, z, c) if b & j: @@ -1161,7 +1161,7 @@ if negativeOutput and z.sign != 0: z = z.sub(c) return z - + @jit.elidable def neg(self): return rbigint(self._digits, -self.sign, self.numdigits()) @@ -1261,7 +1261,7 @@ else: return NULLRBIGINT - + hishift = SHIFT - loshift z = rbigint([NULLDIGIT] * newsize, self.sign, newsize) i = 0 @@ -1286,10 +1286,10 @@ wordshift = int_other / SHIFT loshift = int_other % SHIFT newsize = self.numdigits() - wordshift - + if newsize <= 0: return NULLRBIGINT - + hishift = SHIFT - loshift z = rbigint([NULLDIGIT] * newsize, self.sign, newsize) i = 0 @@ -1301,11 +1301,11 @@ newdigit |= (self.udigit(wordshift+1) << hishift) z.setdigit(i, newdigit) i += 1 - wordshift += 1 + wordshift += 1 z._normalize() return z rshift._always_inline_ = 'try' # It's so fast that it's always benefitial. - + @jit.elidable def abs_rshift_and_mask(self, bigshiftcount, mask): assert isinstance(bigshiftcount, r_ulonglong) @@ -1642,7 +1642,7 @@ if adigit == bdigit: return NULLRBIGINT - + return rbigint.fromint(adigit - bdigit) z = rbigint([NULLDIGIT] * size_a, 1, size_a) @@ -1744,7 +1744,7 @@ carry >>= SHIFT j = UDIGIT_TYPE(0) while j < size_b1: - # this operation does not overflow using + # this operation does not overflow using # SHIFT = (LONG_BIT // 2) - 1 = B - 1; in fact before it # carry and z.widedigit(pz) are less than 2**(B - 1); # b.widedigit(j + 1) * f0 < (2**(B-1) - 1)**2; so @@ -1987,7 +1987,7 @@ """Multiply by a single digit and add a single digit, ignoring the sign. """ assert n > 0 - + size_a = a.numdigits() z = rbigint([NULLDIGIT] * (size_a+1), 1) assert extra & MASK == extra @@ -2071,7 +2071,7 @@ wm1 = w.widedigit(abs(size_w-1)) wm2 = w.widedigit(abs(size_w-2)) - + j = size_v - 1 k -= 1 while k >= 0: From pypy.commits at gmail.com Mon Feb 5 07:10:10 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:10:10 -0800 (PST) Subject: [pypy-commit] pypy default: argh!!! Fix 0l % 0l and 0l % 0 to raise ZeroDivisionError :-( Message-ID: <5a7849a2.02c21c0a.6721d.b4e6@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93761:deaebc8ec57a Date: 2018-02-05 12:52 +0100 http://bitbucket.org/pypy/pypy/changeset/deaebc8ec57a/ Log: argh!!! Fix 0l % 0l and 0l % 0 to raise ZeroDivisionError :-( diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -782,10 +782,12 @@ @jit.elidable def mod(self, other): + if other.sign == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT - if other.sign != 0 and other.numdigits() == 1: + if other.numdigits() == 1: digit = other.digit(0) if digit == 1: return NULLRBIGINT @@ -819,6 +821,8 @@ @jit.elidable def int_mod(self, other): + if other == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT @@ -826,7 +830,7 @@ # Fallback to long. return self.mod(rbigint.fromint(other)) - elif other != 0: + if 1: # preserve indentation to preserve history digit = abs(other) if digit == 1: return NULLRBIGINT @@ -852,8 +856,6 @@ if rem == 0: return NULLRBIGINT mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1) - else: - raise ZeroDivisionError("long division or modulo by zero") if mod.sign * (-1 if other < 0 else 1) == -1: mod = mod.int_add(other) diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -131,11 +131,12 @@ def test_mod(self): for op1 in gen_signs(long_vals): + rl_op1 = rbigint.fromlong(op1) for op2 in gen_signs(long_vals): + rl_op2 = rbigint.fromlong(op2) if not op2: + py.test.raises(ZeroDivisionError, rl_op1.mod, rl_op2) continue - rl_op1 = rbigint.fromlong(op1) - rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.mod(rl_op2) r2 = op1 % op2 @@ -143,10 +144,11 @@ def test_int_mod(self): for x in gen_signs(long_vals): + op1 = rbigint.fromlong(x) for y in signed_int_vals: if not y: + py.test.raises(ZeroDivisionError, op1.int_mod, 0) continue - op1 = rbigint.fromlong(x) r1 = op1.int_mod(y) r2 = x % y assert r1.tolong() == r2 From pypy.commits at gmail.com Mon Feb 5 07:10:12 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:10:12 -0800 (PST) Subject: [pypy-commit] pypy default: remove copy-pasted code between mod and int_mod Message-ID: <5a7849a4.d4951c0a.8645.448f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93762:8ce3c7c2a330 Date: 2018-02-05 12:01 +0100 http://bitbucket.org/pypy/pypy/changeset/8ce3c7c2a330/ Log: remove copy-pasted code between mod and int_mod diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -788,31 +788,9 @@ return NULLRBIGINT if other.numdigits() == 1: - digit = other.digit(0) - if digit == 1: - return NULLRBIGINT - elif digit == 2: - modm = self.digit(0) & 1 - if modm: - return ONENEGATIVERBIGINT if other.sign == -1 else ONERBIGINT - return NULLRBIGINT - elif digit & (digit - 1) == 0: - mod = self.int_and_(digit - 1) - else: - # Perform - size = self.numdigits() - 1 - if size > 0: - rem = self.widedigit(size) - size -= 1 - while size >= 0: - rem = ((rem << SHIFT) + self.widedigit(size)) % digit - size -= 1 - else: - rem = self.digit(0) % digit - - if rem == 0: - return NULLRBIGINT - mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1) + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_mod(otherint) else: div, mod = _divrem(self, other) if mod.sign * other.sign == -1: @@ -844,6 +822,7 @@ else: # Perform size = self.numdigits() - 1 + if size > 0: rem = self.widedigit(size) size -= 1 From pypy.commits at gmail.com Mon Feb 5 07:10:14 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:10:14 -0800 (PST) Subject: [pypy-commit] pypy default: remove code duplication in floordiv Message-ID: <5a7849a6.5088df0a.2fc51.f36a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93763:864485a54ece Date: 2018-02-05 12:27 +0100 http://bitbucket.org/pypy/pypy/changeset/864485a54ece/ Log: remove code duplication in floordiv diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -762,12 +762,10 @@ @jit.elidable def floordiv(self, other): - if self.sign == 1 and other.numdigits() == 1 and other.sign == 1: - digit = other.digit(0) - if digit == 1: - return rbigint(self._digits[:self.size], 1, self.size) - elif digit and digit & (digit - 1) == 0: - return self.rshift(ptwotable[digit]) + if other.numdigits() == 1: + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_floordiv(otherint) div, mod = _divrem(self, other) if mod.sign * other.sign == -1: From pypy.commits at gmail.com Mon Feb 5 07:10:16 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:10:16 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge default Message-ID: <5a7849a8.06d1df0a.a1991.53a9@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93764:3ee3f44df079 Date: 2018-02-05 13:08 +0100 http://bitbucket.org/pypy/pypy/changeset/3ee3f44df079/ Log: merge default diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -831,10 +831,12 @@ @jit.elidable def mod(self, other): + if other.sign == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT - if other.sign != 0 and other.numdigits() == 1: + if other.numdigits() == 1: otherint = other.digit(0) * other.sign assert int_in_valid_range(otherint) return self.int_mod(otherint) @@ -846,6 +848,8 @@ @jit.elidable def int_mod(self, other): + if other == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT @@ -853,7 +857,7 @@ # Fallback to long. return self.mod(rbigint.fromint(other)) - elif other != 0: + if 1: # preserve indentation to preserve history digit = abs(other) if digit == 1: return NULLRBIGINT @@ -880,8 +884,6 @@ if rem == 0: return NULLRBIGINT mod = rbigint([rem], -1 if self.sign < 0 else 1, 1) - else: - raise ZeroDivisionError("long division or modulo by zero") if mod.sign * (-1 if other < 0 else 1) == -1: mod = mod.int_add(other) diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -165,11 +165,12 @@ def test_mod(self): for op1 in gen_signs(long_vals): + rl_op1 = rbigint.fromlong(op1) for op2 in gen_signs(long_vals): + rl_op2 = rbigint.fromlong(op2) if not op2: + py.test.raises(ZeroDivisionError, rl_op1.mod, rl_op2) continue - rl_op1 = rbigint.fromlong(op1) - rl_op2 = rbigint.fromlong(op2) r1 = rl_op1.mod(rl_op2) r2 = op1 % op2 @@ -177,10 +178,11 @@ def test_int_mod(self): for x in gen_signs(long_vals): + op1 = rbigint.fromlong(x) for y in signed_int_vals: if not y: + py.test.raises(ZeroDivisionError, op1.int_mod, 0) continue - op1 = rbigint.fromlong(x) r1 = op1.int_mod(y) r2 = x % y assert r1.tolong() == r2 From pypy.commits at gmail.com Mon Feb 5 07:25:10 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:25:10 -0800 (PST) Subject: [pypy-commit] pypy default: Backed out changeset 864485a54ece Message-ID: <5a784d26.c79cdf0a.55686.9248@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93765:124c875dc8d6 Date: 2018-02-05 13:22 +0100 http://bitbucket.org/pypy/pypy/changeset/124c875dc8d6/ Log: Backed out changeset 864485a54ece diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -762,10 +762,12 @@ @jit.elidable def floordiv(self, other): - if other.numdigits() == 1: - otherint = other.digit(0) * other.sign - assert int_in_valid_range(otherint) - return self.int_floordiv(otherint) + if self.sign == 1 and other.numdigits() == 1 and other.sign == 1: + digit = other.digit(0) + if digit == 1: + return rbigint(self._digits[:self.size], 1, self.size) + elif digit and digit & (digit - 1) == 0: + return self.rshift(ptwotable[digit]) div, mod = _divrem(self, other) if mod.sign * other.sign == -1: From pypy.commits at gmail.com Mon Feb 5 07:25:13 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 05 Feb 2018 04:25:13 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge default Message-ID: <5a784d29.c83f1c0a.577e2.ce45@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r93766:6ccab3e62413 Date: 2018-02-05 13:23 +0100 http://bitbucket.org/pypy/pypy/changeset/6ccab3e62413/ Log: merge default From pypy.commits at gmail.com Tue Feb 6 04:50:27 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 01:50:27 -0800 (PST) Subject: [pypy-commit] pypy memory-accounting: merge default Message-ID: <5a797a63.a490df0a.acae4.cf86@mx.google.com> Author: fijal Branch: memory-accounting Changeset: r93767:e8d1f3223393 Date: 2018-02-05 16:34 +0100 http://bitbucket.org/pypy/pypy/changeset/e8d1f3223393/ Log: merge default diff too long, truncating to 2000 out of 90615 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -44,3 +44,10 @@ d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,2 +1,3 @@ pytest hypothesis +vmprof diff --git a/pypy/module/test_lib_pypy/test_json_extra.py b/extra_tests/test_json.py rename from pypy/module/test_lib_pypy/test_json_extra.py rename to extra_tests/test_json.py --- a/pypy/module/test_lib_pypy/test_json_extra.py +++ b/extra_tests/test_json.py @@ -1,4 +1,6 @@ -import py, json +import pytest +import json +from hypothesis import given, strategies def is_(x, y): return type(x) is type(y) and x == y @@ -6,12 +8,26 @@ def test_no_ensure_ascii(): assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') assert is_(json.dumps("\xc0", ensure_ascii=False), '"\xc0"') - e = py.test.raises(UnicodeDecodeError, json.dumps, - (u"\u1234", "\xc0"), ensure_ascii=False) - assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ") - e = py.test.raises(UnicodeDecodeError, json.dumps, - ("\xc0", u"\u1234"), ensure_ascii=False) - assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps((u"\u1234", "\xc0"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps(("\xc0", u"\u1234"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") def test_issue2191(): assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,7 +12,8 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, int): + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,6 +7,16 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -17,10 +17,13 @@ """ from __future__ import division -import time as _time +import time as _timemodule import math as _math import struct as _struct +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + _SENTINEL = object() def _cmp(x, y): @@ -179,7 +182,7 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. @@ -247,7 +250,7 @@ else: push(ch) newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) + return _timemodule.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): @@ -433,7 +436,7 @@ raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) -class timedelta(object): +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -489,7 +492,7 @@ if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -667,7 +670,7 @@ timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) -class date(object): +class date(dateinterop): """Concrete date type. Constructors: @@ -707,12 +710,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -724,13 +727,13 @@ @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t) @classmethod @@ -1061,7 +1064,7 @@ _tzinfo_class = tzinfo -class time(object): +class time(timeinterop): """Time with time zone. Constructors: @@ -1097,14 +1100,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1408,15 +1411,20 @@ if isinstance(year, bytes) and len(year) == 10 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1461,7 +1469,7 @@ A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - converter = _time.localtime if tz is None else _time.gmtime + converter = _timemodule.localtime if tz is None else _timemodule.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) @@ -1470,7 +1478,7 @@ @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - return cls._from_timestamp(_time.gmtime, t, None) + return cls._from_timestamp(_timemodule.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): @@ -1488,18 +1496,18 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." - t = _time.time() + t = _timemodule.time() return cls.utcfromtimestamp(t) @classmethod @@ -1797,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1805,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -20,6 +20,7 @@ or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" __metaclass__ = _structseq.structseqtype + name = "resource.struct_rusage" ru_utime = _structseq.structseqfield(0, "user time used") ru_stime = _structseq.structseqfield(1, "system time used") diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -149,7 +149,7 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X:: +On Mac OS X: Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -355,7 +355,11 @@ containers (as list items or in sets for example), the exact rule of equality used is "``if x is y or x == y``" (on both CPython and PyPy); as a consequence, because all ``nans`` are identical in PyPy, you -cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__). +Another consequence is that ``cmp(float('nan'), float('nan')) == 0``, because +``cmp`` checks with ``is`` first whether the arguments are identical (there is +no good value to return from this call to ``cmp``, because ``cmp`` pretends +that there is a total order on floats, but that is wrong for NaNs). .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of @@ -541,6 +545,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,31 +1,38 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:d56dadcef996 - -.. branch: cppyy-packaging -Cleanup and improve cppyy packaging - -.. branch: docs-osx-brew-openssl - -.. branch: keep-debug-symbols -Add a smartstrip tool, which can optionally keep the debug symbols in a -separate file, instead of just stripping them away. Use it in packaging - -.. branch: bsd-patches -Fix failures on FreeBSD, contributed by David Naylor as patches on the issue -tracker (issues 2694, 2695, 2696, 2697) - -.. branch: run-extra-tests -Run extra_tests/ in buildbot - -.. branch: vmprof-0.4.10 -Upgrade the _vmprof backend to vmprof 0.4.10 - -.. branch: fix-vmprof-stacklet-switch -Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) - -.. branch: win32-vcvars - +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -0,0 +1,42 @@ +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst --- a/pypy/doc/whatsnew-pypy2-5.6.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst @@ -101,7 +101,7 @@ .. branch: newinitwarn -Match CPython's stricter handling of __new/init__ arguments +Match CPython's stricter handling of ``__new__``/``__init__`` arguments .. branch: openssl-1.1 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -11,7 +11,7 @@ To build pypy-c you need a working python environment, and a C compiler. It is possible to translate with a CPython 2.6 or later, but this is not -the preferred way, because it will take a lot longer to run � depending +the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -103,6 +103,7 @@ must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the ``...\9.0\VC`` directory, and edit it, changing the lines that set ``VCINSTALLDIR`` and ``WindowsSdkDir``:: + set VCINSTALLDIR=%~dp0\ set WindowsSdkDir=%~dp0\..\WinSDK\ diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import we_are_translated from pypy.interpreter.astcompiler import ast, misc, symtable @@ -266,7 +266,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -276,9 +276,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1096,17 +1096,21 @@ encoding = self.compile_info.encoding flags = self.compile_info.flags unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - try: - sub_strings_w = [parsestring.parsestr(space, encoding, atom_node.get_child(i).get_value(), - unicode_literals) - for i in range(atom_node.num_children())] - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise self.error('(unicode error) %s' % errmsg, atom_node) + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), + unicode_literals)) + except error.OperationError as e: + if not e.match(space, space.w_UnicodeError): + raise + # UnicodeError in literal: turn into SyntaxError + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + if child is None: + child = atom_node + raise self.error('(unicode error) %s' % errmsg, child) # This implements implicit string concatenation. if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1246,3 +1246,15 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-1: truncated \\xXX escape") + input = "u'\\x1'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,6 +208,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,4 +1,7 @@ -from pypy.interpreter.unicodehelper import encode_utf8, decode_utf8 +import pytest +import struct +from pypy.interpreter.unicodehelper import ( + encode_utf8, decode_utf8, unicode_encode_utf_32_be) class FakeSpace: pass @@ -24,3 +27,23 @@ assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] + + at pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) +def test_utf32_surrogates(unich): + assert (unicode_encode_utf_32_be(unich, 1, None) == + struct.pack('>i', ord(unich))) + with pytest.raises(UnicodeEncodeError): + unicode_encode_utf_32_be(unich, 1, None, allow_surrogates=False) + + def replace_with(ru, rs): + def errorhandler(errors, enc, msg, u, startingpos, endingpos): + if errors == 'strict': + raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg) + return ru, rs, endingpos + return unicode_encode_utf_32_be( + u"<%s>" % unich, 3, None, + errorhandler, allow_surrogates=False) + + assert replace_with(u'rep', None) == u''.encode('utf-32-be') + assert (replace_with(None, '\xca\xfe\xca\xfe') == + '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,7 +1,11 @@ +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder +from rpython.rlib import runicode +from rpython.rlib.runicode import ( + default_unicode_error_encode, default_unicode_error_decode, + MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR) from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import specialize -from rpython.rlib import runicode -from pypy.module._codecs import interp_codecs @specialize.memo() def decode_error_handler(space): @@ -37,6 +41,7 @@ # These functions take and return unwrapped rpython strings and unicodes def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result, consumed = runicode.str_decode_unicode_escape( @@ -71,3 +76,229 @@ uni, len(uni), "strict", errorhandler=None, allow_surrogates=True) + +# ____________________________________________________________ +# utf-32 + +def str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native") + return result, length + +def str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big") + return result, length + +def str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little") + return result, length + +def py3k_str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2) + return result, length + +def py3k_str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be') + return result, length + +def py3k_str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le') + return result, length + +BOM32_DIRECT = intmask(0x0000FEFF) +BOM32_REVERSE = intmask(0xFFFE0000) + +def str_decode_utf_32_helper(s, size, errors, final=True, + errorhandler=None, + byteorder="native", + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_decode + bo = 0 + + if BYTEORDER == 'little': + iorder = [0, 1, 2, 3] + else: + iorder = [3, 2, 1, 0] + + # Check for BOM marks (U+FEFF) in the input and adjust current + # byte order setting accordingly. In native mode, the leading BOM + # mark is skipped, in all other modes, it is copied to the output + # stream as-is (giving a ZWNBSP character). + pos = 0 + if byteorder == 'native': + if size >= 4: + bom = intmask( + (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | + (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) + if BYTEORDER == 'little': + if bom == BOM32_DIRECT: + pos += 4 + bo = -1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = 1 + else: + if bom == BOM32_DIRECT: + pos += 4 + bo = 1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = -1 + elif byteorder == 'little': + bo = -1 + else: + bo = 1 + if size == 0: + return u'', 0, bo + if bo == -1: + # force little endian + iorder = [0, 1, 2, 3] + elif bo == 1: + # force big endian + iorder = [3, 2, 1, 0] + + result = UnicodeBuilder(size // 4) + + while pos < size: + # remaining bytes at the end? (size should be divisible by 4) + if len(s) - pos < 4: + if not final: + break + r, pos = errorhandler(errors, public_encoding_name, + "truncated data", + s, pos, len(s)) + result.append(r) + if len(s) - pos < 4: + break + continue + ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) | + (ord(s[pos + iorder[1]]) << 8) | ord(s[pos + iorder[0]])) + if ch >= 0x110000: + r, pos = errorhandler(errors, public_encoding_name, + "codepoint not in range(0x110000)", + s, pos, len(s)) + result.append(r) + continue + + if MAXUNICODE < 65536 and ch >= 0x10000: + ch -= 0x10000L + result.append(unichr(0xD800 + (ch >> 10))) + result.append(unichr(0xDC00 + (ch & 0x03FF))) + else: + result.append(UNICHR(ch)) + pos += 4 + return result.build(), pos, bo + +def _STORECHAR32(result, CH, byteorder): + c0 = chr(((CH) >> 24) & 0xff) + c1 = chr(((CH) >> 16) & 0xff) + c2 = chr(((CH) >> 8) & 0xff) + c3 = chr((CH) & 0xff) + if byteorder == 'little': + result.append(c3) + result.append(c2) + result.append(c1) + result.append(c0) + else: + result.append(c0) + result.append(c1) + result.append(c2) + result.append(c3) + +def unicode_encode_utf_32_helper(s, size, errors, + errorhandler=None, + allow_surrogates=True, + byteorder='little', + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_encode + if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() + return "" + + result = StringBuilder(size * 4 + 4) + if byteorder == 'native': + _STORECHAR32(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 + ch2 = 0 + if not allow_surrogates and 0xD800 <= ch < 0xE000: + ru, rs, pos = errorhandler( + errors, public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + if rs is not None: + # py3k only + if len(rs) % 4 != 0: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + result.append(rs) + continue + for ch in ru: + if ord(ch) < 0xD800: + _STORECHAR32(result, ord(ch), byteorder) + else: + errorhandler( + 'strict', public_encoding_name, + 'surrogates not allowed', s, pos - 1, pos) + continue + if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size: + ch2 = ord(s[pos]) + if 0xDC00 <= ch2 < 0xE000: + ch = (((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000 + pos += 1 + _STORECHAR32(result, ch, byteorder) + + return result.build() + +def unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native") + +def unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big") + +def unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little") + +def py3k_unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native", + 'utf-32-' + BYTEORDER2) + +def py3k_unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big", + 'utf-32-be') + +def py3k_unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little", + 'utf-32-le') diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -2,11 +2,13 @@ Interp-level implementation of the basic space operations. """ +import math + from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from rpython.rlib.runicode import UNICHR -from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away +from rpython.rlib.rfloat import isfinite, round_double, round_away from rpython.rlib import rfloat import __builtin__ @@ -151,7 +153,7 @@ else: # finite x, and ndigits is not unreasonably large z = round_double(number, ndigits) - if isinf(z): + if math.isinf(z): raise oefmt(space.w_OverflowError, "rounded value too large to represent") return space.newfloat(z) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -404,6 +404,7 @@ def test_cmp(self): + assert cmp(float('nan'), float('nan')) == 0 assert cmp(9,9) == 0 assert cmp(0,9) < 0 assert cmp(9,0) > 0 diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -58,6 +58,14 @@ } +class PyPyDateTime(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + } + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -108,6 +116,7 @@ "thread": ThreadModule, "intop": IntOpModule, "os": OsModule, + '_pypydatetime': PyPyDateTime, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -0,0 +1,24 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from rpython.tool.sourcetools import func_with_new_name + +def create_class(name): + class W_Class(W_Root): + 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + def descr_new__(space, w_type): + return space.allocate_instance(W_Class, w_type) + + W_Class.typedef = TypeDef(name, + __new__ = interp2app(func_with_new_name( + W_Class.descr_new__.im_func, + '%s_new' % (name,))), + ) + W_Class.typedef.acceptable_as_base_class = True + return W_Class + +W_DateTime_Time = create_class('pypydatetime_time') +W_DateTime_Date = create_class('pypydatetime_date') +W_DateTime_Delta = create_class('pypydatetime_delta') + + diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, + w_extra_compile_args=None): try: import cffi from cffi import FFI # <== the system one, which @@ -55,10 +56,14 @@ sources = [] if w_extra_source is not None: sources.append(space.str_w(w_extra_source)) + kwargs = {} + if w_extra_compile_args is not None: + kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args) ext = ffiplatform.get_extension(c_file, module_name, include_dirs=[str(rdir)], export_symbols=['_cffi_pypyinit_' + base_module_name], - sources=sources) + sources=sources, + **kwargs) ffiplatform.compile(str(rdir), ext) for extension in ['so', 'pyd', 'dylib']: @@ -2054,3 +2059,51 @@ "Such structs are only supported as return value if the function is " "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()" "+ffibuilder.set_source() and not taking a final '...' argument)") + + def test_gcc_visibility_hidden(self): + import sys + if sys.platform == 'win32': + skip("test for gcc/clang") + ffi, lib = self.prepare(""" + int f(int); + """, "test_gcc_visibility_hidden", """ + int f(int a) { return a + 40; } + """, extra_compile_args=['-fvisibility=hidden']) + assert lib.f(2) == 42 + + def test_override_default_definition(self): + ffi, lib = self.prepare(""" + typedef long int16_t, char16_t; + """, "test_override_default_definition", """ + """) + assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long") + + def test_char16_char32_plain_c(self): + ffi, lib = self.prepare(""" + char16_t foo_2bytes(char16_t); + char32_t foo_4bytes(char32_t); + """, "test_char16_char32_type_nocpp", """ + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) + typedef uint_least16_t char16_t; + typedef uint_least32_t char32_t; + #endif + + char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); } + char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); } + """, min_version=(1, 11, 0)) + assert lib.foo_2bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\U00012345') == u'\U0001236f' + raises(TypeError, lib.foo_2bytes, u'\U00012345') + raises(TypeError, lib.foo_2bytes, 1234) + raises(TypeError, lib.foo_4bytes, 1234) + + def test_loader_spec(self): + import sys + ffi, lib = self.prepare("", "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1,10 +1,12 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import we_are_translated, not_rpython from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib import runicode from rpython.rlib.runicode import code_to_unichr, MAXUNICODE from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter import unicodehelper class VersionTag(object): @@ -210,7 +212,8 @@ def xmlcharrefreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -236,7 +239,8 @@ def backslashreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -363,19 +367,23 @@ raise oefmt(space.w_TypeError, "handler must be callable") # ____________________________________________________________ -# delegation to runicode +# delegation to runicode/unicodehelper -from rpython.rlib import runicode +def _find_implementation(impl_name): + try: + func = getattr(unicodehelper, impl_name) + except AttributeError: + func = getattr(runicode, impl_name) + return func def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(uni=unicode, errors='text_or_none') def wrap_encoder(space, uni, errors="strict"): if errors is None: errors = 'strict' state = space.fromcache(CodecState) - func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) return space.newtuple([space.newbytes(result), space.newint(len(uni))]) wrap_encoder.func_name = rname @@ -383,7 +391,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(string='bufferstr', errors='text_or_none', w_final=WrappedDefault(False)) def wrap_decoder(space, string, errors="strict", w_final=None): @@ -391,7 +399,6 @@ errors = 'strict' final = space.is_true(w_final) state = space.fromcache(CodecState) - func = getattr(runicode, rname) result, consumed = func(string, len(string), errors, final, state.decode_error_handler) return space.newtuple([space.newunicode(result), space.newint(consumed)]) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -115,10 +115,10 @@ raises(TypeError, charmap_decode, '\xff', "strict", {0xff: 0x110000}) assert (charmap_decode("\x00\x01\x02", "strict", {0: 0x10FFFF, 1: ord('b'), 2: ord('c')}) == - u"\U0010FFFFbc", 3) + (u"\U0010FFFFbc", 3)) assert (charmap_decode("\x00\x01\x02", "strict", {0: u'\U0010FFFF', 1: u'b', 2: u'c'}) == - u"\U0010FFFFbc", 3) + (u"\U0010FFFFbc", 3)) def test_escape_decode_errors(self): from _codecs import escape_decode as decode @@ -537,8 +537,12 @@ assert '\xff'.decode('utf-7', 'ignore') == '' assert '\x00'.decode('unicode-internal', 'ignore') == '' - def test_backslahreplace(self): - assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000' + def test_backslashreplace(self): + sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" + expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + assert sin.encode('ascii', 'backslashreplace') == expected + expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff" + assert sin.encode("iso-8859-15", "backslashreplace") == expected def test_badhandler(self): import codecs @@ -592,11 +596,11 @@ def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) - return (u"\x01", 1) + return (u"\x01", 4) codecs.register_error("test.hui", handler_unicodeinternal) res = "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui") if sys.maxunicode > 65535: - assert res == u"\u0000\u0001\u0000" # UCS4 build + assert res == u"\u0000\u0001" # UCS4 build else: assert res == u"\x00\x00\x01\x00\x00" # UCS2 build @@ -750,3 +754,31 @@ assert _codecs.unicode_escape_decode(b) == (u'', 0) assert _codecs.raw_unicode_escape_decode(b) == (u'', 0) assert _codecs.unicode_internal_decode(b) == (u'', 0) + + def test_xmlcharrefreplace(self): + r = u'\u1234\u0080\u2345\u0079\u00AB'.encode('latin1', 'xmlcharrefreplace') + assert r == 'ሴ\x80⍅y\xab' + r = u'\u1234\u0080\u2345\u0079\u00AB'.encode('ascii', 'xmlcharrefreplace') + assert r == 'ሴ€⍅y«' + + def test_errorhandler_collection(self): + import _codecs + errors = [] + def record_error(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + errors.append(exc.object[exc.start:exc.end]) + return (u'', exc.end) + _codecs.register_error("test.record", record_error) + + sin = u"\xac\u1234\u1234\u20ac\u8000" + assert sin.encode("ascii", "test.record") == "" + assert errors == [sin] + + errors = [] + assert sin.encode("latin-1", "test.record") == "\xac" + assert errors == [u'\u1234\u1234\u20ac\u8000'] + + errors = [] + assert sin.encode("iso-8859-15", "test.record") == "\xac\xa4" + assert errors == [u'\u1234\u1234', u'\u8000'] diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -1,5 +1,6 @@ from rpython.rlib.rstacklet import StackletThread from rpython.rlib import jit +from rpython.rlib import rvmprof from pypy.interpreter.error import OperationError, get_cleared_operation_error from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.baseobjspace import W_Root @@ -222,12 +223,15 @@ self.h = h global_state.clear() try: + rvmprof.start_sampling() frame = self.bottomframe w_result = frame.execute_frame() except Exception as e: global_state.propagate_exception = e else: global_state.w_value = w_result + finally: + rvmprof.stop_sampling() self.sthread.ec.topframeref = jit.vref_None global_state.origin = self global_state.destination = self diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/_continuation/test/test_greenlet.py rename from pypy/module/test_lib_pypy/test_greenlet.py rename to pypy/module/_continuation/test/test_greenlet.py diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py --- a/pypy/module/_continuation/test/test_stacklet.py +++ b/pypy/module/_continuation/test/test_stacklet.py @@ -1,7 +1,10 @@ +import pytest import os +from rpython.rlib.rvmprof.test.support import fakevmprof +from pypy.interpreter.gateway import interp2app from pypy.module._continuation.test.support import BaseAppTest - + at pytest.mark.usefixtures('app_fakevmprof') class AppTestStacklet(BaseAppTest): def setup_class(cls): BaseAppTest.setup_class.im_func(cls) @@ -34,10 +37,34 @@ return res return stack """) + cls.w_appdirect = cls.space.wrap(cls.runappdirect) if cls.runappdirect: # make sure that "self.stack" does not pass the self cls.w_stack = staticmethod(cls.w_stack.im_func) + + @pytest.fixture + def app_fakevmprof(self, fakevmprof): + """ + This is automaticaly re-initialized for every method: thanks to + fakevmprof's finalizer, it checks that we called {start,stop}_sampling + the in pairs + """ + w = self.space.wrap + i2a = interp2app + def is_sampling_enabled(space): + return space.wrap(fakevmprof.is_sampling_enabled) + self.w_is_sampling_enabled = w(i2a(is_sampling_enabled)) + # + def start_sampling(space): + fakevmprof.start_sampling() + self.w_start_sampling = w(i2a(start_sampling)) + # + def stop_sampling(space): + fakevmprof.stop_sampling() + self.w_stop_sampling = w(i2a(stop_sampling)) + + def test_new_empty(self): from _continuation import continulet # @@ -770,3 +797,25 @@ continulet.switch(c1, to=c2) raises(error, continulet.switch, c1, to=c2) + + def test_sampling_inside_callback(self): + if self.appdirect: + # see also + # extra_tests.test_vmprof_greenlet.test_sampling_inside_callback + # for a "translated" version of this test + skip("we can't run this until we have _vmprof.is_sampling_enabled") + from _continuation import continulet + # + def my_callback(c1): + assert self.is_sampling_enabled() + return 42 + # + try: + self.start_sampling() + assert self.is_sampling_enabled() + c = continulet(my_callback) + res = c.switch() + assert res == 42 + assert self.is_sampling_enabled() + finally: + self.stop_sampling() diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -1,4 +1,5 @@ import py +import pytest try: import _continuation except ImportError: @@ -101,11 +102,7 @@ particular, we need to ensure that vmprof does not sample the stack in the middle of a switch, else we read nonsense. """ - try: - import _vmprof - except ImportError: - py.test.skip("no _vmprof") - # + _vmprof = pytest.importorskip('_vmprof') def switch_forever(c): while True: c.switch() From pypy.commits at gmail.com Tue Feb 6 04:50:31 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 01:50:31 -0800 (PST) Subject: [pypy-commit] pypy default: Merge memory-accounting which adds extra functions to gc that Message-ID: <5a797a67.f487df0a.69fbc.feb9@mx.google.com> Author: fijal Branch: Changeset: r93768:f23eec5d0d6d Date: 2018-02-06 10:49 +0100 http://bitbucket.org/pypy/pypy/changeset/f23eec5d0d6d/ Log: Merge memory-accounting which adds extra functions to gc that let you describe the whole memory diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -35,7 +35,7 @@ except ImportError: assert '__pypy__' not in sys.builtin_module_names newlist_hint = lambda sizehint: [] - add_memory_pressure = lambda size: None + add_memory_pressure = lambda size, obj: None if sys.version_info[0] >= 3: StandardError = Exception @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) - add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, + res = factory(database, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements) + add_memory_pressure(100 * 1024, res) + return res def _unicode_text_factory(x): diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -142,11 +142,14 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate, w_obj=None): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" + #if space.is_none(w_obj): rgc.add_memory_pressure(estimate) + #else: + # rgc.add_memory_pressure(estimate, w_obj) @unwrap_spec(w_frame=PyFrame) def locals_to_fast(space, w_frame): diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -61,7 +61,8 @@ ctx = ropenssl.EVP_MD_CTX_new() if ctx is None: raise MemoryError - rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size, + self) try: if copy_from: if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from): diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1316,8 +1316,9 @@ if not ctx: raise ssl_error(space, "failed to allocate SSL context") - rgc.add_memory_pressure(10 * 1024 * 1024) self = space.allocate_instance(_SSLContext, w_subtype) + assert isinstance(self, _SSLContext) + rgc.add_memory_pressure(10 * 1024 * 1024, self) self.ctx = ctx self.check_hostname = False self.register_finalizer(space) diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -19,6 +19,7 @@ space.config.translation.gctransformer == "framework"): self.appleveldefs.update({ 'dump_rpy_heap': 'app_referents.dump_rpy_heap', + 'get_stats': 'app_referents.get_stats', }) self.interpleveldefs.update({ 'get_rpy_roots': 'referents.get_rpy_roots', @@ -28,6 +29,7 @@ 'get_objects': 'referents.get_objects', 'get_referents': 'referents.get_referents', 'get_referrers': 'referents.get_referrers', + '_get_stats': 'referents.get_stats', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -48,3 +48,49 @@ file.flush() fd = file.fileno() gc._dump_rpy_heap(fd) + +class GcStats(object): + def __init__(self, s): + self._s = s + for item in ('total_gc_memory', 'jit_backend_used', + 'total_memory_pressure', + 'total_allocated_memory', 'jit_backend_allocated', + 'peak_memory', 'peak_allocated_memory'): + setattr(self, item, self._format(getattr(self._s, item))) + self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + + self._s.jit_backend_used) + self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + + self._s.jit_backend_allocated) + + def _format(self, v): + if v < 1000000: + # bit unlikely ;-) + return "%.1fkB" % (v / 1024.) + return "%.1fMB" % (v / 1024. / 1024.) + + def repr(self): + return """Total memory consumed: +GC used: %s (peak: %s) +raw assembler used: %s +memory pressure: %s +----------------------------- +Total: %s + +Total memory allocated: +GC allocated: %s (peak: %s) +raw assembler allocated: %s +memory pressure: %s +----------------------------- +Total: %s +""" % (self.total_gc_memory, self.peak_memory, + self.jit_backend_used, + self.total_memory_pressure, + self.memory_used_sum, + + self.total_allocated_memory, self.peak_allocated_memory, + self.jit_backend_allocated, + self.total_memory_pressure, + self.memory_allocated_sum) + +def get_stats(): + return GcStats(gc._get_stats()) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -1,7 +1,7 @@ -from rpython.rlib import rgc +from rpython.rlib import rgc, jit_hooks from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.error import oefmt, wrap_oserror from rpython.rlib.objectmodel import we_are_translated @@ -170,3 +170,33 @@ l = rgc.get_typeids_list() list_w = [space.newint(l[i]) for i in range(len(l))] return space.newlist(list_w) + +class W_GcStats(W_Root): + def __init__(self): + self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY) + self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY) + self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY) + self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY) + self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None) + self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None) + +W_GcStats.typedef = TypeDef("GcStats", + total_memory_pressure=interp_attrproperty("total_memory_pressure", + cls=W_GcStats, wrapfn="newint"), + total_gc_memory=interp_attrproperty("total_gc_memory", + cls=W_GcStats, wrapfn="newint"), + peak_allocated_memory=interp_attrproperty("peak_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + peak_memory=interp_attrproperty("peak_memory", + cls=W_GcStats, wrapfn="newint"), + total_allocated_memory=interp_attrproperty("total_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + jit_backend_allocated=interp_attrproperty("jit_backend_allocated", + cls=W_GcStats, wrapfn="newint"), + jit_backend_used=interp_attrproperty("jit_backend_used", + cls=W_GcStats, wrapfn="newint"), +) + +def get_stats(space): + return W_GcStats() diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -840,11 +840,11 @@ # Currently this is just the size of the pointer and some estimated bytes. # The struct isn't actually defined in expat.h - it is in xmlparse.c # XXX: find a good estimate of the XML_ParserStruct - rgc.add_memory_pressure(XML_Parser_SIZE + 300) if not xmlparser: raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed") parser = W_XMLParserType(space, xmlparser, w_intern) + rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser) XML_SetUnknownEncodingHandler( parser.itself, UnknownEncodingHandlerData_callback, rffi.cast(rffi.VOIDP, parser.id)) diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -71,6 +71,7 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() + self.memory_pressure_types = set() self.register_builtins() diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -675,6 +675,8 @@ def _ll_1_gc_add_memory_pressure(num): llop.gc_add_memory_pressure(lltype.Void, num) + def _ll_2_gc_add_memory_pressure(num, obj): + llop.gc_add_memory_pressure(lltype.Void, num, obj) def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None): diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -83,7 +83,9 @@ has_custom_trace, fast_path_tracing, has_gcptr, - cannot_pin): + cannot_pin, + has_memory_pressure, + get_memory_pressure_ofs): self.finalizer_handlers = finalizer_handlers self.destructor_or_custom_trace = destructor_or_custom_trace self.is_old_style_finalizer = is_old_style_finalizer @@ -103,6 +105,8 @@ self.fast_path_tracing = fast_path_tracing self.has_gcptr = has_gcptr self.cannot_pin = cannot_pin + self.has_memory_pressure = has_memory_pressure + self.get_memory_pressure_ofs = get_memory_pressure_ofs def get_member_index(self, type_id): return self.member_index(type_id) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -72,6 +72,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize +from rpython.rlib import rgc from rpython.memory.gc.minimarkpage import out_of_memory # @@ -371,6 +372,7 @@ self.old_rawmalloced_objects = self.AddressStack() self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) + self.rawmalloced_peak_size = r_uint(0) self.gc_state = STATE_SCANNING # @@ -996,6 +998,8 @@ # Record the newly allocated object and its full malloced size. # The object is young or old depending on the argument. self.rawmalloced_total_size += r_uint(allocsize) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) if alloc_young: if not self.young_rawmalloced_objects: self.young_rawmalloced_objects = self.AddressDict() @@ -1023,7 +1027,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): # Decrement by 'sizehint' plus a very little bit extra. This # is needed e.g. for _rawffi, which may allocate a lot of tiny # arrays. @@ -1183,6 +1187,24 @@ """ return self.ac.total_memory_used + self.rawmalloced_total_size + def get_total_memory_alloced(self): + """ Return the total memory allocated + """ + return self.ac.total_memory_alloced + self.rawmalloced_total_size + + def get_peak_memory_alloced(self): + """ Return the peak memory ever allocated. The peaks + can be at different times, but we just don't worry for now + """ + return self.ac.peak_memory_alloced + self.rawmalloced_peak_size + + def get_peak_memory_used(self): + """ Return the peak memory GC felt ever responsible for + """ + mem_allocated = max(self.ac.peak_memory_used, + self.ac.total_memory_used) + return mem_allocated + self.rawmalloced_peak_size + def threshold_reached(self, extra=0): return (self.next_major_collection_threshold - float(self.get_total_memory_used())) < float(extra) @@ -2155,6 +2177,8 @@ # size_gc_header = self.gcheaderbuilder.size_gc_header self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize)) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) self.old_rawmalloced_objects.append(arena + size_gc_header) return arena @@ -2932,6 +2956,21 @@ self.old_objects_with_weakrefs.delete() self.old_objects_with_weakrefs = new_with_weakref + def get_stats(self, stats_no): + from rpython.memory.gc import inspector + + if stats_no == rgc.TOTAL_MEMORY: + return intmask(self.get_total_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_MEMORY: + return intmask(self.get_peak_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_ALLOCATED_MEMORY: + return intmask(self.get_peak_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY: + return intmask(self.get_total_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_MEMORY_PRESSURE: + return inspector.count_memory_pressure(self) + return 0 + # ---------- # RawRefCount diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py --- a/rpython/memory/gc/inspector.py +++ b/rpython/memory/gc/inspector.py @@ -2,6 +2,7 @@ Utility RPython functions to inspect objects in the GC. """ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.objectmodel import free_non_gc_object from rpython.rlib import rposix, rgc, jit @@ -92,17 +93,12 @@ AddressStack = get_address_stack() -class HeapDumper(object): - _alloc_flavor_ = "raw" - BUFSIZE = 8192 # words +class BaseWalker(object): + _alloc_flavor_ = 'raw' - def __init__(self, gc, fd): + def __init__(self, gc): self.gc = gc self.gcflag = gc.gcflag_extra - self.fd = rffi.cast(rffi.INT, fd) - self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, - flavor='raw') - self.buf_count = 0 if self.gcflag == 0: self.seen = AddressDict() self.pending = AddressStack() @@ -111,8 +107,107 @@ if self.gcflag == 0: self.seen.delete() self.pending.delete() + free_non_gc_object(self) + + def add_roots(self): + self.gc.enumerate_all_roots(_hd_add_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.walk(pendingroots) + pendingroots.delete() + self.end_add_roots_marker() + + def end_add_roots_marker(self): + pass + + def add(self, obj): + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) + + def walk(self, pending): + while pending.non_empty(): + self.processobj(pending.pop()) + + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unobj(self, obj): + gc = self.gc + gc.trace(obj, self._unref, None) + + def _unref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unobj(pending.pop()) + + def finish_processing(self): + if self.gcflag != 0: + self.clear_gcflag_again() + self.unwalk(self.pending) + + def process(self): + self.add_roots() + self.walk(self.pending) + + +class MemoryPressureCounter(BaseWalker): + + def __init__(self, gc): + self.count = 0 + BaseWalker.__init__(self, gc) + + def processobj(self, obj): + gc = self.gc + typeid = gc.get_type_id(obj) + if gc.has_memory_pressure(typeid): + ofs = gc.get_memory_pressure_ofs(typeid) + val = (obj + ofs).signed[0] + self.count += val + gc.trace(obj, self._ref, None) + + def _ref(self, pointer, _): + obj = pointer.address[0] + self.add(obj) + + +class HeapDumper(BaseWalker): + BUFSIZE = 8192 # words + + def __init__(self, gc, fd): + BaseWalker.__init__(self, gc) + self.fd = rffi.cast(rffi.INT, fd) + self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, + flavor='raw') + self.buf_count = 0 + + def delete(self): lltype.free(self.writebuffer, flavor='raw') - free_non_gc_object(self) + BaseWalker.delete(self) @jit.dont_look_inside def flush(self): @@ -143,6 +238,7 @@ self.write(0) self.write(0) self.write(-1) + end_add_roots_marker = write_marker def writeobj(self, obj): gc = self.gc @@ -152,64 +248,13 @@ self.write(gc.get_size_incl_hash(obj)) gc.trace(obj, self._writeref, None) self.write(-1) + processobj = writeobj def _writeref(self, pointer, _): obj = pointer.address[0] self.write(llmemory.cast_adr_to_int(obj)) self.add(obj) - def add(self, obj): - if self.gcflag == 0: - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) - else: - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) == 0: - hdr.tid |= self.gcflag - self.pending.append(obj) - - def add_roots(self): - self.gc.enumerate_all_roots(_hd_add_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.walk(pendingroots) - pendingroots.delete() - self.write_marker() - - def walk(self, pending): - while pending.non_empty(): - self.writeobj(pending.pop()) - - # ---------- - # A simplified copy of the above, to make sure we walk again all the - # objects to clear the 'gcflag'. - - def unwriteobj(self, obj): - gc = self.gc - gc.trace(obj, self._unwriteref, None) - - def _unwriteref(self, pointer, _): - obj = pointer.address[0] - self.unadd(obj) - - def unadd(self, obj): - assert self.gcflag != 0 - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) != 0: - hdr.tid &= ~self.gcflag - self.pending.append(obj) - - def clear_gcflag_again(self): - self.gc.enumerate_all_roots(_hd_unadd_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.unwalk(pendingroots) - pendingroots.delete() - - def unwalk(self, pending): - while pending.non_empty(): - self.unwriteobj(pending.pop()) def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) @@ -219,15 +264,20 @@ def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) - heapdumper.add_roots() - heapdumper.walk(heapdumper.pending) + heapdumper.process() heapdumper.flush() - if heapdumper.gcflag != 0: - heapdumper.clear_gcflag_again() - heapdumper.unwalk(heapdumper.pending) + heapdumper.finish_processing() heapdumper.delete() return True +def count_memory_pressure(gc): + counter = MemoryPressureCounter(gc) + counter.process() + counter.finish_processing() + res = counter.count + counter.delete() + return res + def get_typeids_z(gc): srcaddress = gc.root_walker.gcdata.typeids_z return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR)) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -828,7 +828,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): self.next_major_collection_threshold -= sizehint if self.next_major_collection_threshold < 0: # cannot trigger a full collection now, but we can ensure diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -141,6 +141,9 @@ # the total memory used, counting every block in use, without # the additional bookkeeping stuff. self.total_memory_used = r_uint(0) + self.peak_memory_used = r_uint(0) + self.total_memory_alloced = r_uint(0) + self.peak_memory_alloced = r_uint(0) def _new_page_ptr_list(self, length): @@ -293,7 +296,11 @@ # # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address - arena_base = llarena.arena_malloc(self.arena_size, False) + arena_base = llarena.arena_mmap(self.arena_size) + self.total_memory_alloced += self.arena_size + self.peak_memory_alloced = max(self.total_memory_alloced, + self.peak_memory_alloced) + if not arena_base: out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size @@ -321,6 +328,8 @@ """Prepare calls to mass_free_incremental(): moves the chained lists into 'self.old_xxx'. """ + self.peak_memory_used = max(self.peak_memory_used, + self.total_memory_used) self.total_memory_used = r_uint(0) # size_class = self.small_request_threshold >> WORD_POWER_2 @@ -397,8 +406,8 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. - llarena.arena_reset(arena.base, self.arena_size, 4) - llarena.arena_free(arena.base) + llarena.arena_munmap(arena.base, self.arena_size) + self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 # diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -392,23 +392,30 @@ inline = True) if getattr(GCClass, 'raw_malloc_memory_pressure', False): - def raw_malloc_memory_pressure_varsize(length, itemsize): + def raw_malloc_memory_pressure_varsize(length, itemsize, adr): totalmem = length * itemsize if totalmem > 0: - gcdata.gc.raw_malloc_memory_pressure(totalmem) + gcdata.gc.raw_malloc_memory_pressure(totalmem, adr) #else: probably an overflow -- the following rawmalloc # will fail then - def raw_malloc_memory_pressure(sizehint): - gcdata.gc.raw_malloc_memory_pressure(sizehint) + def raw_malloc_memory_pressure(sizehint, adr): + gcdata.gc.raw_malloc_memory_pressure(sizehint, adr) self.raw_malloc_memory_pressure_varsize_ptr = getfn( raw_malloc_memory_pressure_varsize, - [annmodel.SomeInteger(), annmodel.SomeInteger()], + [annmodel.SomeInteger(), annmodel.SomeInteger(), + SomeAddress()], annmodel.s_None, minimal_transform = False) self.raw_malloc_memory_pressure_ptr = getfn( raw_malloc_memory_pressure, - [annmodel.SomeInteger()], + [annmodel.SomeInteger(), SomeAddress()], annmodel.s_None, minimal_transform = False) + if getattr(GCClass, 'get_stats', False): + def get_stats(stats_no): + return gcdata.gc.get_stats(stats_no) + self.get_stats_ptr = getfn(get_stats, [annmodel.SomeInteger()], + annmodel.SomeInteger()) + self.identityhash_ptr = getfn(GCClass.identityhash.im_func, [s_gc, s_gcref], @@ -831,6 +838,39 @@ gct_fv_gc_malloc_varsize = gct_fv_gc_malloc + def gct_gc_add_memory_pressure(self, hop): + def _find_correct_type(TP): + T = TP.TO + while 'special_memory_pressure' not in T._flds: + T = T._flds['super'] + return T + + if hasattr(self, 'raw_malloc_memory_pressure_ptr'): + op = hop.spaceop + size = op.args[0] + if len(op.args) == 2: + v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure") + T = _find_correct_type(op.args[1].concretetype) + v_inst = hop.genop("cast_pointer", [op.args[1]], + resulttype=lltype.Ptr(T)) + hop.genop("bare_setfield", [v_inst, v_fld, size]) + v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]], + resulttype=llmemory.Address) + else: + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) + hop.genop("direct_call", [self.raw_malloc_memory_pressure_ptr, + size, v_adr]) + + + def gct_gc_get_stats(self, hop): + if hasattr(self, 'get_stats_ptr'): + return hop.genop("direct_call", + [self.get_stats_ptr, hop.spaceop.args[0]], + resultvar=hop.spaceop.result) + hop.genop("same_as", [rmodel.inputconst(lltype.Signed, 0)], + resultvar=hop.spaceop.result) + + def gct_gc__collect(self, hop): op = hop.spaceop if len(op.args) == 1: diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -535,12 +535,7 @@ return self.varsize_malloc_helper(hop, flags, meth, []) def gct_gc_add_memory_pressure(self, hop): - if hasattr(self, 'raw_malloc_memory_pressure_ptr'): - op = hop.spaceop - size = op.args[0] - return hop.genop("direct_call", - [self.raw_malloc_memory_pressure_ptr, - size]) + pass def varsize_malloc_helper(self, hop, flags, meth, extraargs): def intconst(c): return rmodel.inputconst(lltype.Signed, c) @@ -574,9 +569,10 @@ c_offset_to_length): if flags.get('add_memory_pressure', False): if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'): + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) hop.genop("direct_call", [self.raw_malloc_memory_pressure_varsize_ptr, - v_length, c_item_size]) + v_length, c_item_size, v_adr]) if c_offset_to_length is None: if flags.get('zero'): fnptr = self.raw_malloc_varsize_no_length_zero_ptr diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -21,13 +21,21 @@ # A destructor is called when the object is about to be freed. # A custom tracer (CT) enumerates the addresses that contain GCREFs. # Both are called with the address of the object as only argument. + # They're embedded in a struct that has raw_memory_offset as another + # argument, which is only valid if T_HAS_MEMORY_PRESSURE is set CUSTOM_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) CUSTOM_FUNC_PTR = lltype.Ptr(CUSTOM_FUNC) + CUSTOM_DATA_STRUCT = lltype.Struct('custom_data', + ('customfunc', CUSTOM_FUNC_PTR), + ('memory_pressure_offset', lltype.Signed), # offset to where the amount + # of owned memory pressure is stored + ) + CUSTOM_DATA_STRUCT_PTR = lltype.Ptr(CUSTOM_DATA_STRUCT) # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", ("infobits", lltype.Signed), # combination of the T_xxx consts - ("customfunc", CUSTOM_FUNC_PTR), + ("customdata", CUSTOM_DATA_STRUCT_PTR), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), hints={'immutable': True}, @@ -81,14 +89,14 @@ def q_cannot_pin(self, typeid): typeinfo = self.get(typeid) ANY = (T_HAS_GCPTR | T_IS_WEAKREF) - return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customfunc) + return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customdata) def q_finalizer_handlers(self): adr = self.finalizer_handlers # set from framework.py or gcwrapper.py return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY)) def q_destructor_or_custom_trace(self, typeid): - return self.get(typeid).customfunc + return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): typeinfo = self.get(typeid) @@ -139,6 +147,15 @@ infobits = self.get(typeid).infobits return infobits & T_ANY_SLOW_FLAG == 0 + def q_has_memory_pressure(self, typeid): + infobits = self.get(typeid).infobits + return infobits & T_HAS_MEMORY_PRESSURE != 0 + + def q_get_memory_pressure_ofs(self, typeid): + infobits = self.get(typeid).infobits + assert infobits & T_HAS_MEMORY_PRESSURE != 0 + return self.get(typeid).customdata.memory_pressure_offset + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -159,7 +176,9 @@ self.q_has_custom_trace, self.q_fast_path_tracing, self.q_has_gcptr, - self.q_cannot_pin) + self.q_cannot_pin, + self.q_has_memory_pressure, + self.q_get_memory_pressure_ofs) def _has_got_custom_trace(self, typeid): type_info = self.get(typeid) @@ -176,8 +195,9 @@ T_HAS_CUSTOM_TRACE = 0x200000 T_HAS_OLDSTYLE_FINALIZER = 0x400000 T_HAS_GCPTR = 0x1000000 -T_KEY_MASK = intmask(0xFE000000) # bug detection only -T_KEY_VALUE = intmask(0x5A000000) # bug detection only +T_HAS_MEMORY_PRESSURE = 0x2000000 # first field is memory pressure field +T_KEY_MASK = intmask(0xFC000000) # bug detection only +T_KEY_VALUE = intmask(0x58000000) # bug detection only def _check_valid_type_info(p): ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id") @@ -192,6 +212,25 @@ ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") +def has_special_memory_pressure(TYPE): + if TYPE._is_varsize(): + return False + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return True + if 'super' not in T._flds: + return False + T = T._flds['super'] + +def get_memory_pressure_ofs(TYPE): + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return llmemory.offsetof(T, 'special_memory_pressure') + if 'super' not in T._flds: + assert False, "get_ and has_memory_pressure disagree" + T = T._flds['super'] def encode_type_shape(builder, info, TYPE, index): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" @@ -202,12 +241,18 @@ infobits |= T_HAS_GCPTR # fptrs = builder.special_funcptr_for_type(TYPE) - if fptrs: + if fptrs or has_special_memory_pressure(TYPE): + customdata = lltype.malloc(GCData.CUSTOM_DATA_STRUCT, flavor='raw', + immortal=True) + info.customdata = customdata if "destructor" in fptrs: - info.customfunc = fptrs["destructor"] + customdata.customfunc = fptrs["destructor"] if "old_style_finalizer" in fptrs: - info.customfunc = fptrs["old_style_finalizer"] + customdata.customfunc = fptrs["old_style_finalizer"] infobits |= T_HAS_OLDSTYLE_FINALIZER + if has_special_memory_pressure(TYPE): + infobits |= T_HAS_MEMORY_PRESSURE + info.customdata.memory_pressure_offset = get_memory_pressure_ofs(TYPE) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -83,9 +83,9 @@ def gettypeid(self, obj): return self.get_type_id(lltype.typeOf(obj).TO) - def add_memory_pressure(self, size): + def add_memory_pressure(self, size, adr): if hasattr(self.gc, 'raw_malloc_memory_pressure'): - self.gc.raw_malloc_memory_pressure(size) + self.gc.raw_malloc_memory_pressure(size, adr) def shrink_array(self, p, smallersize): if hasattr(self.gc, 'shrink_array'): diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -598,21 +598,31 @@ return False return type(x).__module__ != '__builtin__' # keep non-builtins -def add_memory_pressure(estimate): +def add_memory_pressure(estimate, object=None): """Add memory pressure for OpaquePtrs.""" pass class AddMemoryPressureEntry(ExtRegistryEntry): _about_ = add_memory_pressure - def compute_result_annotation(self, s_nbytes): + def compute_result_annotation(self, s_nbytes, s_object=None): from rpython.annotator import model as annmodel + if s_object is not None: + if not isinstance(s_object, annmodel.SomeInstance): + raise Exception("Wrong kind of object passed to " + "add memory pressure") + self.bookkeeper.memory_pressure_types.add(s_object.classdef) return annmodel.s_None def specialize_call(self, hop): - [v_size] = hop.inputargs(lltype.Signed) + v_size = hop.inputarg(lltype.Signed, 0) + if len(hop.args_v) == 2: + v_obj = hop.inputarg(hop.args_r[1], 1) + args = [v_size, v_obj] + else: + args = [v_size] hop.exception_cannot_occur() - return hop.genop('gc_add_memory_pressure', [v_size], + return hop.genop('gc_add_memory_pressure', args, resulttype=lltype.Void) @@ -640,6 +650,15 @@ else: return id(gcref._x) +(TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, + PEAK_MEMORY, PEAK_ALLOCATED_MEMORY) = range(5) + + at not_rpython +def get_stats(stat_no): + """ Long docstring goes here + """ + raise NotImplementedError + @not_rpython def dump_rpy_heap(fd): raise NotImplementedError @@ -834,6 +853,18 @@ return hop.genop('gc_get_rpy_type_index', vlist, resulttype = hop.r_result) +class Entry(ExtRegistryEntry): + _about_ = get_stats + def compute_result_annotation(self, s_no): + from rpython.annotator.model import SomeInteger + if not isinstance(s_no, SomeInteger): + raise Exception("expecting an integer") + return SomeInteger() + def specialize_call(self, hop): + args = hop.inputargs(lltype.Signed) + hop.exception_cannot_occur() + return hop.genop('gc_get_stats', args, resulttype=lltype.Signed) + @not_rpython def _is_rpy_instance(gcref): raise NotImplementedError diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -85,7 +85,11 @@ def allocate_lock(): - return Lock(allocate_ll_lock()) + # Add some memory pressure for the size of the lock because it is an + # Opaque object + lock = Lock(allocate_ll_lock()) + rgc.add_memory_pressure(TLOCKP_SIZE, lock) + return lock @specialize.arg(0) def ll_start_new_thread(func): @@ -248,9 +252,6 @@ if rffi.cast(lltype.Signed, res) <= 0: lltype.free(ll_lock, flavor='raw', track_allocation=False) raise error("out of resources") - # Add some memory pressure for the size of the lock because it is an - # Opaque object - rgc.add_memory_pressure(TLOCKP_SIZE) return ll_lock def free_ll_lock(ll_lock): diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -269,7 +269,6 @@ compress data. """ stream = lltype.malloc(z_stream, flavor='raw', zero=True) - rgc.add_memory_pressure(rffi.sizeof(z_stream)) err = _deflateInit2(stream, level, method, wbits, memLevel, strategy) if err == Z_OK: if zdict is not None: @@ -304,7 +303,6 @@ decompress data. """ stream = lltype.malloc(z_stream, flavor='raw', zero=True) - rgc.add_memory_pressure(rffi.sizeof(z_stream)) err = _inflateInit2(stream, wbits) if err == Z_OK: if zdict is not None and wbits < 0: diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -327,6 +327,17 @@ assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() +def arena_mmap(nbytes): + """Allocate and return a new arena, zero-initialized by the + system, calling mmap().""" + return arena_malloc(nbytes, True) + +def arena_munmap(arena_addr, nbytes): + """Release an arena allocated with arena_mmap().""" + arena_free(arena_addr) + assert nbytes == arena_addr.arena.nbytes + + def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. This can also be used on a subrange of the arena. @@ -530,6 +541,31 @@ llfakeimpl=arena_free, sandboxsafe=True) +def llimpl_arena_mmap(nbytes): + from rpython.rlib import rmmap + flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS + prot = rmmap.PROT_READ | rmmap.PROT_WRITE + p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( + lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) + if p == rffi.cast(llmemory.Address, -1): + p = rffi.cast(llmemory.Address, 0) + return p +register_external(arena_mmap, [int], llmemory.Address, + 'll_arena.arena_mmap', + llimpl=llimpl_arena_mmap, + llfakeimpl=arena_mmap, + sandboxsafe=True) + +def llimpl_arena_munmap(arena_addr, nbytes): + from rpython.rlib import rmmap + assert nbytes >= 0 + rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) +register_external(arena_munmap, [llmemory.Address, int], None, + 'll_arena.arena_munmap', + llimpl=llimpl_arena_munmap, + llfakeimpl=arena_munmap, + sandboxsafe=True) + def llimpl_arena_reset(arena_addr, size, zero): if zero: if zero == 1: diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -485,6 +485,7 @@ 'gc_gettypeid' : LLOp(), 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), + 'gc_get_stats' : LLOp(), 'gc_fq_next_dead' : LLOp(), 'gc_fq_register' : LLOp(), 'gc_ignore_finalizer' : LLOp(canrun=True), diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -15,6 +15,7 @@ RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, Void, FuncType, Bool, Signed, functionptr, attachRuntimeTypeInfo) from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.llannotation import lltype_to_annotation from rpython.rtyper.llannotation import SomePtr from rpython.rtyper.lltypesystem import rstr from rpython.rtyper.rmodel import ( @@ -475,6 +476,13 @@ self.lowleveltype = Ptr(self.object_type) self.gcflavor = gcflavor + def has_special_memory_pressure(self, tp): + if 'special_memory_pressure' in tp._flds: + return True + if 'super' in tp._flds: + return self.has_special_memory_pressure(tp._flds['super']) + return False + def _setup_repr(self, llfields=None, hints=None, adtmeths=None): # NOTE: don't store mutable objects like the dicts below on 'self' # before they are fully built, to avoid strange bugs in case @@ -523,6 +531,16 @@ if not attrdef.readonly and self.is_quasi_immutable(name): llfields.append(('mutate_' + name, OBJECTPTR)) + bookkeeper = self.rtyper.annotator.bookkeeper + if self.classdef in bookkeeper.memory_pressure_types: + # we don't need to add it if it's already there for some of + # the parent type + if not self.has_special_memory_pressure(self.rbase.object_type): + llfields.append(('special_memory_pressure', lltype.Signed)) + fields['special_memory_pressure'] = ( + 'special_memory_pressure', + self.rtyper.getrepr(lltype_to_annotation(lltype.Signed))) + object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -663,6 +681,8 @@ while base.classdef is not None: base = base.rbase for fieldname in base.fields: + if fieldname == 'special_memory_pressure': + continue try: mangled, r = base._get_field(fieldname) except KeyError: @@ -717,6 +737,9 @@ resulttype=Ptr(self.object_type)) ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) self.setfield(vptr, '__class__', ctypeptr, llops) + if self.has_special_memory_pressure(self.object_type): + self.setfield(vptr, 'special_memory_pressure', + inputconst(lltype.Signed, 0), llops) # initialize instance attributes from their defaults from the class if self.classdef is not None: flds = self.allinstancefields.keys() diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py --- a/rpython/translator/backendopt/writeanalyze.py +++ b/rpython/translator/backendopt/writeanalyze.py @@ -65,6 +65,11 @@ elif op.opname == "gc_store_indexed": if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]): return self._gc_store_indexed_result(op) + elif op.opname == 'gc_add_memory_pressure': + # special_memory_pressure would be overwritten by zero, because + # the JIT cannot see the field write, which is why we assume + # it can write anything + return top_set return empty_set def _array_result(self, TYPE): diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1613,7 +1613,7 @@ digest = ropenssl.EVP_get_digestbyname('sha1') self.ctx = ropenssl.EVP_MD_CTX_new() ropenssl.EVP_DigestInit(self.ctx, digest) - rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64, self) def __del__(self): ropenssl.EVP_MD_CTX_free(self.ctx) @@ -1624,12 +1624,16 @@ am3 = am2 am2 = am1 am1 = A() + am1 = am2 = am3 = None # what can we use for the res? - return 0 + for i in range(10): + gc.collect() + return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) return f def test_nongc_opaque_attached_to_gc(self): res = self.run("nongc_opaque_attached_to_gc") + # the res is 0 for non-memory-pressure-accounting GC assert res == 0 def define_limited_memory(self): @@ -1668,6 +1672,38 @@ class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcpolicy = "incminimark" + def define_total_memory_pressure(cls): + class A(object): + def __init__(self): + rgc.add_memory_pressure(10, self) + + def __del__(self): + pass + + class B(A): + def __init__(self): + rgc.add_memory_pressure(10, self) + + class C(A): + pass + + class Glob(object): + pass + glob = Glob() + + def f(): + glob.l = [None] * 3 + for i in range(10000): + glob.l[i % 3] = A() + glob.l[(i + 1) % 3] = B() + glob.l[(i + 2) % 3] = C() + return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + return f + + def test_total_memory_pressure(self): + res = self.run("total_memory_pressure") + assert res == 30 # total reachable is 3 + def define_random_pin(self): class A: foo = None From pypy.commits at gmail.com Tue Feb 6 04:52:22 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 01:52:22 -0800 (PST) Subject: [pypy-commit] pypy default: document the branhc Message-ID: <5a797ad6.421d1c0a.cd220.9e9d@mx.google.com> Author: fijal Branch: Changeset: r93769:25b0174b027f Date: 2018-02-06 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/25b0174b027f/ Log: document the branhc diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -36,3 +36,7 @@ .. branch: 2634_datetime_timedelta_performance Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory From pypy.commits at gmail.com Tue Feb 6 08:40:32 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 05:40:32 -0800 (PST) Subject: [pypy-commit] pypy default: cleanup Message-ID: <5a79b050.148edf0a.279a1.7edd@mx.google.com> Author: fijal Branch: Changeset: r93770:d655e2839a95 Date: 2018-02-06 14:38 +0100 http://bitbucket.org/pypy/pypy/changeset/d655e2839a95/ Log: cleanup diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -35,7 +35,7 @@ except ImportError: assert '__pypy__' not in sys.builtin_module_names newlist_hint = lambda sizehint: [] - add_memory_pressure = lambda size, obj: None + add_memory_pressure = lambda size: None if sys.version_info[0] >= 3: StandardError = Exception @@ -155,7 +155,7 @@ # backed by :memory: or a file) res = factory(database, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements) - add_memory_pressure(100 * 1024, res) + add_memory_pressure(100 * 1024) return res diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -142,14 +142,11 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(space, estimate, w_obj=None): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" - #if space.is_none(w_obj): rgc.add_memory_pressure(estimate) - #else: - # rgc.add_memory_pressure(estimate, w_obj) @unwrap_spec(w_frame=PyFrame) def locals_to_fast(space, w_frame): diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -269,6 +269,7 @@ compress data. """ stream = lltype.malloc(z_stream, flavor='raw', zero=True) + rgc.add_memory_pressure(rffi.sizeof(stream)) err = _deflateInit2(stream, level, method, wbits, memLevel, strategy) if err == Z_OK: if zdict is not None: @@ -303,6 +304,7 @@ decompress data. """ stream = lltype.malloc(z_stream, flavor='raw', zero=True) + rgc.add_memory_pressure(rffi.sizeof(z_stream)) err = _inflateInit2(stream, wbits) if err == Z_OK: if zdict is not None and wbits < 0: From pypy.commits at gmail.com Tue Feb 6 08:40:34 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 05:40:34 -0800 (PST) Subject: [pypy-commit] pypy default: add optionality about memory pressure, defaulting to False Message-ID: <5a79b052.93b2df0a.3f114.f583@mx.google.com> Author: fijal Branch: Changeset: r93771:f59f5814a43c Date: 2018-02-06 14:39 +0100 http://bitbucket.org/pypy/pypy/changeset/f59f5814a43c/ Log: add optionality about memory pressure, defaulting to False diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -69,28 +69,51 @@ return "%.1fMB" % (v / 1024. / 1024.) def repr(self): - return """Total memory consumed: -GC used: %s (peak: %s) -raw assembler used: %s -memory pressure: %s ------------------------------ -Total: %s + if self._s.total_memory_pressure != -1: + return """Total memory consumed: + GC used: %s (peak: %s) + raw assembler used: %s + memory pressure: %s + ----------------------------- + Total: %s -Total memory allocated: -GC allocated: %s (peak: %s) -raw assembler allocated: %s -memory pressure: %s ------------------------------ -Total: %s -""" % (self.total_gc_memory, self.peak_memory, - self.jit_backend_used, - self.total_memory_pressure, - self.memory_used_sum, + Total memory allocated: + GC allocated: %s (peak: %s) + raw assembler allocated: %s + memory pressure: %s + ----------------------------- + Total: %s + """ % (self.total_gc_memory, self.peak_memory, + self.jit_backend_used, + self.total_memory_pressure, + self.memory_used_sum, - self.total_allocated_memory, self.peak_allocated_memory, - self.jit_backend_allocated, - self.total_memory_pressure, - self.memory_allocated_sum) + self.total_allocated_memory, self.peak_allocated_memory, + self.jit_backend_allocated, + self.total_memory_pressure, + self.memory_allocated_sum) + else: + return """Total memory consumed: + GC used: %s (peak: %s) + raw assembler used: %s + ----------------------------- + Total: %s + + Total memory allocated: + GC allocated: %s (peak: %s) + raw assembler allocated: %s + memory pressure: %s + ----------------------------- + Total: %s + """ % (self.total_gc_memory, self.peak_memory, + self.jit_backend_used, + self.memory_used_sum, + + self.total_allocated_memory, self.peak_allocated_memory, + self.jit_backend_allocated, + self.total_memory_pressure, + self.memory_allocated_sum) + def get_stats(): return GcStats(gc._get_stats()) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -172,8 +172,11 @@ return space.newlist(list_w) class W_GcStats(W_Root): - def __init__(self): - self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + def __init__(self, memory_pressure): + if memory_pressure: + self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + else: + self.total_memory_pressure = -1 self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY) self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY) self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY) @@ -198,5 +201,6 @@ cls=W_GcStats, wrapfn="newint"), ) -def get_stats(space): - return W_GcStats() + at unwrap_spec(memory_pressure=bool) +def get_stats(space, memory_pressure=False): + return W_GcStats(memory_pressure) From pypy.commits at gmail.com Tue Feb 6 10:09:30 2018 From: pypy.commits at gmail.com (fijal) Date: Tue, 06 Feb 2018 07:09:30 -0800 (PST) Subject: [pypy-commit] pypy default: never move changes by hand... Message-ID: <5a79c52a.46bcdf0a.4f604.3102@mx.google.com> Author: fijal Branch: Changeset: r93772:d209cc859e9b Date: 2018-02-06 16:08 +0100 http://bitbucket.org/pypy/pypy/changeset/d209cc859e9b/ Log: never move changes by hand... diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -269,7 +269,7 @@ compress data. """ stream = lltype.malloc(z_stream, flavor='raw', zero=True) - rgc.add_memory_pressure(rffi.sizeof(stream)) + rgc.add_memory_pressure(rffi.sizeof(z_stream)) err = _deflateInit2(stream, level, method, wbits, memLevel, strategy) if err == Z_OK: if zdict is not None: From pypy.commits at gmail.com Tue Feb 6 14:05:55 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 11:05:55 -0800 (PST) Subject: [pypy-commit] pypy winapi: WIP adapting _winapi.py to _winapi.c from CPython Message-ID: <5a79fc93.6ba9df0a.c2730.62bc@mx.google.com> Author: Matti Picus Branch: winapi Changeset: r93774:8266092b4620 Date: 2018-02-02 19:02 +0100 http://bitbucket.org/pypy/pypy/changeset/8266092b4620/ Log: WIP adapting _winapi.py to _winapi.c from CPython diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -63,6 +63,12 @@ HANDLE hStdError; } STARTUPINFO, *LPSTARTUPINFO; +typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; +} SECURITY_ATTRIBUTES, *PSECURITY_ATTRIBUTES, *LPSECURITY_ATTRIBUTES; + typedef struct { HANDLE hProcess; HANDLE hThread; @@ -70,9 +76,41 @@ DWORD dwThreadId; } PROCESS_INFORMATION, *LPPROCESS_INFORMATION; +typedef struct _OVERLAPPED { + ULONG_PTR Internal; + ULONG_PTR InternalHigh; + union { + struct { + DWORD Offset; + DWORD OffsetHigh; + } DUMMYSTRUCTNAME; + PVOID Pointer; + } DUMMYUNIONNAME; + + HANDLE hEvent; +} OVERLAPPED, *LPOVERLAPPED; + + DWORD WINAPI GetVersion(void); BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD); +HANDLE WINAPI CreateNamedPipeA(LPCSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateNamedPipeW(LPWSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateFileA(LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD); +BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED); +HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR); +HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR); +VOID WINAPI SetEvent(HANDLE); +BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED); BOOL WINAPI CloseHandle(HANDLE); +DWORD WINAPI GetLastError(VOID); +BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL); + HANDLE WINAPI GetCurrentProcess(void); BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0), - _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), - _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xAB\x03\x00\x00\x13\x11\x00\x00\xB0\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\xAA\x03\x00\x00\xA8\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x29\x11\x00\x00\x18\x03\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x6B\x03\x00\x00\x49\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x33\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x66\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\xA9\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xAB\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x6E\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6B\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x77\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xB0\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\x05\x09\x00\x00\x03\x09\x00\x00\x02\x01\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xAF\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x27\x23CancelIoEx',0,b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x27\x23ConnectNamedPipe',0,b'\x00\x00\x6D\x23CreateEventA',0,b'\x00\x00\x73\x23CreateEventW',0,b'\x00\x00\x79\x23CreateFileA',0,b'\x00\x00\x9B\x23CreateFileW',0,b'\x00\x00\x82\x23CreateNamedPipeA',0,b'\x00\x00\x91\x23CreateNamedPipeW',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x48\x23CreateProcessW',0,b'\x00\x00\x3F\x23DuplicateHandle',0,b'\x00\x00\x8F\x23GetCurrentProcess',0,b'\x00\x00\x35\x23GetExitCodeProcess',0,b'\x00\x00\x63\x23GetLastError',0,b'\x00\x00\x5E\x23GetModuleFileNameW',0,b'\x00\x00\x2B\x23GetOverlappedResult',0,b'\x00\x00\x8C\x23GetStdHandle',0,b'\x00\x00\x63\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x57\x23SetErrorMode',0,b'\x00\x00\xA4\x23SetEvent',0,b'\x00\x00\x39\x23SetNamedPipeHandleState',0,b'\x00\x00\x31\x23TerminateProcess',0,b'\x00\x00\x5A\x23WaitForSingleObject',0,b'\x00\x00\x54\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x68\x23_getwch',0,b'\x00\x00\x68\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x6A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x65\x23_ungetwch',0), + _struct_unions = ((b'\x00\x00\x00\xAD\x00\x00\x00\x03$1',b'\x00\x00\xAC\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x00\xAC\x00\x00\x00\x02$2',b'\x00\x00\x18\x11Offset',b'\x00\x00\x18\x11OffsetHigh'),(b'\x00\x00\x00\xA8\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\xAA\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x66\x11wShowWindow',b'\x00\x00\x66\x11cbReserved2',b'\x00\x00\xAE\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x00\xA7\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x18\x11Internal',b'\x00\x00\x18\x11InternalHigh',b'\x00\x00\xAD\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x00\xA9\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x18\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle')), + _typenames = (b'\x00\x00\x00\x29LPOVERLAPPED',b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x6ELPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\xA7OVERLAPPED',b'\x00\x00\x00\xA8PROCESS_INFORMATION',b'\x00\x00\x00\x6EPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xA9SECURITY_ATTRIBUTES',b'\x00\x00\x00\xAASTARTUPINFO',b'\x00\x00\x00\x66wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -1,12 +1,12 @@ """ -Support routines for subprocess module. +Support routines for subprocess and multiprocess module. Currently, this extension module is only required when using the -subprocess module on Windows. +modules on Windows. """ import sys if sys.platform != 'win32': - raise ImportError("The '_subprocess' module is only available on Windows") + raise ImportError("The '_winapi' module is only available on Windows") # Declare external Win32 functions @@ -14,7 +14,7 @@ _kernel32 = _ffi.dlopen('kernel32') GetVersion = _kernel32.GetVersion - +NULL = _ffi.NULL # Now the _subprocess module implementation @@ -33,13 +33,116 @@ def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") - res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size) + res = _kernel32.CreatePipe(handles, handles + 1, NULL, size) if not res: raise _WinError() return _handle2int(handles[0]), _handle2int(handles[1]) +def CreateNamedPipe(*args): + handle = _kernel32.CreateNamedPipeW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def CreateFile(*args): + handle = _kernel32.CreateFileW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def SetNamedPipeHandleState(namedpipe, mode, max_collection_count, collect_data_timeout): + d0 = _ffi.new('DWORD[1]', [mode]) + if max_collection_count is None: + d1 = NULL + else: + d1 = _ffi.new('DWORD[1]', [max_collection_count]) + if collect_data_timeout is None: + d2 = NULL + else: + d2 = _ffi.new('DWORD[1]', [collect_data_timeout]) + ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2) + if not ret: + raise _WinError() + +class Overlapped(object): + def __init__(self, handle): + self.overlapped = _ffi.new('OVERLAPPED[1]') + self.handle = handle + self.readbuffer = None + self.pending = 0 + self.completed = 0 + self.writebuffer = None + self.overlapped[0].hEvent = \ + _kernel32.CreateEventW(NULL, True, False, NULL) + + def __del__(self): + # do this somehow else + xxx + err = _kernel32.GetLastError() + bytes = _ffi.new('DWORD[1]') + o = overlapped[0] + if overlapped[0].pending: + if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ + self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): + # The operation is no longer pending, nothing to do + pass + else: + raise RuntimeError('deleting an overlapped strucwith a pending operation not supported') + + @property + def event(self): + return None + + def GetOverlappedResult(self, wait): + transferred = _ffi.new('DWORD[1]', [0]) + res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) + if not res: + res = GetLastError() + if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED): + self.completed = 1 + self.pending = 0 + elif res == ERROR_IO_INCOMPLETE: + pass + else: + self.pending = 0 + raise _WinError() + if self.completed and self.read_buffer: + if transferred != len(self.read_buffer): + raise _WinError() + return transferred[0], err + + def getbuffer(self): + xxx + return None + + def cancel(self): + xxx + return None + + +def ConnectNamedPipe(handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + if overlapped: + # Overlapped ConnectNamedPipe never returns a success code + assert success == 0 + err = _kernel32.GetLastError() + if err == ERROR_IO_PENDING: + overlapped[0].pending = 1 + elif err == ERROR_PIPE_CONNECTED: + _kernel32.SetEvent(ov.overlapped[0].hEvent) + else: + del ov + raise _WinError() + return ov + elif not success: + raise _WinError() + def GetCurrentProcess(): return _handle2int(_kernel32.GetCurrentProcess()) @@ -155,6 +258,7 @@ raise _WinError() return _ffi.string(buf) +# #define macros from WinBase.h and elsewhere STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -171,3 +275,52 @@ CREATE_UNICODE_ENVIRONMENT = 0x400 STILL_ACTIVE = 259 _MAX_PATH = 260 + +ERROR_SUCCESS = 0 +ERROR_NETNAME_DELETED = 64 +ERROR_BROKEN_PIPE = 109 +ERROR_MORE_DATA = 234 +ERROR_PIPE_CONNECTED = 535 +ERROR_OPERATION_ABORTED = 995 +ERROR_IO_INCOMPLETE = 996 +ERROR_IO_PENDING = 997 + +PIPE_ACCESS_INBOUND = 0x00000001 +PIPE_ACCESS_OUTBOUND = 0x00000002 +PIPE_ACCESS_DUPLEX = 0x00000003 +PIPE_WAIT = 0x00000000 +PIPE_NOWAIT = 0x00000001 +PIPE_READMODE_BYTE = 0x00000000 +PIPE_READMODE_MESSAGE = 0x00000002 +PIPE_TYPE_BYTE = 0x00000000 +PIPE_TYPE_MESSAGE = 0x00000004 +PIPE_ACCEPT_REMOTE_CLIENTS = 0x00000000 +PIPE_REJECT_REMOTE_CLIENTS = 0x00000008 + +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +GENERIC_EXECUTE= 0x20000000 +GENERIC_ALL = 0x10000000 +INVALID_HANDLE_VALUE = -1 +FILE_FLAG_WRITE_THROUGH = 0x80000000 +FILE_FLAG_OVERLAPPED = 0x40000000 +FILE_FLAG_NO_BUFFERING = 0x20000000 +FILE_FLAG_RANDOM_ACCESS = 0x10000000 +FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 +FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 +FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 +FILE_FLAG_POSIX_SEMANTICS = 0x01000000 +FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +FILE_FLAG_OPEN_NO_RECALL = 0x00100000 +FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000 + +NMPWAIT_WAIT_FOREVER = 0xffffffff +NMPWAIT_NOWAIT = 0x00000001 +NMPWAIT_USE_DEFAULT_WAIT = 0x00000000 + +CREATE_NEW = 1 +CREATE_ALWAYS = 2 +OPEN_EXISTING = 3 +OPEN_ALWAYS = 4 +TRUNCATE_EXISTING = 5 + From pypy.commits at gmail.com Tue Feb 6 14:05:57 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 11:05:57 -0800 (PST) Subject: [pypy-commit] pypy msvc14: make validate_fd into a context manager FdValidator(fd) for msvc Message-ID: <5a79fc95.52bbdf0a.3fc60.98b9@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93775:2f7ecead192b Date: 2018-02-06 12:02 +0100 http://bitbucket.org/pypy/pypy/changeset/2f7ecead192b/ Log: make validate_fd into a context manager FdValidator(fd) for msvc diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'get_hidden_tb' : 'interp_magic.get_hidden_tb', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -107,14 +107,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - def get_console_cp(space): from rpython.rlib import rwin32 # Windows only return space.newtuple([ diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import is_valid_fd, FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -96,25 +96,31 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) + with FdValidator(c_fileno(fp)): + return _feof(fp) def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + try: + with FdValidator(c_fileno(fp)): + return 1 + except IOError: + return 0 pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,7 +1,6 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import sys @@ -80,25 +79,29 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen + if osname == 'posix': # POSIX: we want to check the file descriptor when fdopen() is called, # not later when we read or write data. So we call fstat(), letting # it raise if fd is invalid. - _validate_fd = posix.fstat + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object + + Return an open file object connected to a file descriptor.""" + posix.fstat(fd) + return _fdopen(fd, mode, buffering) + else: - _validate_fd = validate_fd + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object -# Capture file.fdopen at import time, as some code replaces -# __builtins__.file with a custom function. -_fdopen = file.fdopen - -def fdopen(fd, mode='r', buffering=-1): - """fdopen(fd [, mode='r' [, buffering]]) -> file_object - - Return an open file object connected to a file descriptor.""" - _validate_fd(fd) - return _fdopen(fd, mode, buffering) - + Return an open file object connected to a file descriptor.""" + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + return _fdopen(fd, mode, buffering) def tmpfile(): """Create a temporary file. diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,11 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ #if defined _MSC_VER && MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { @@ -96,12 +96,45 @@ errno = EBADF; return 0; } - #else - RPY_EXTERN int - _PyVerify_fd(int fd) + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL}; + RPY_EXTERN void exit_suppress_iph(void*) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) { return 1; } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + void* enter_suppress_iph(void) {return (void*) NULL}; + void exit_suppress_iph(void*) {}; #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] @@ -201,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): - # CPython Issue #11395, PyPy Issue #2636: the Windows console - # returns an error (12: not enough space error) on writing into - # stdout if stdout mode is binary and the length is greater than - # 66,000 bytes (or less, depending on heap usage). Can't easily - # test that, because we need 'fd' to be non-redirected... - count = 32767 - elif count > 0x7fffffff: - count = 0x7fffffff - return count -else: - def is_valid_fd(fd): - return 1 - - @enforceargs(int) - def validate_fd(fd): - pass - - def _bound_for_write(fd, count): - return count - -def closerange(fd_low, fd_high): - # this behaves like os.closerange() from Python 2.6. - for fd in xrange(fd_low, fd_high): - try: - if is_valid_fd(fd): - os.close(fd) - except OSError: - pass - if _WIN32: includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] @@ -266,11 +255,80 @@ if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'): includes.append('sys/ttycom.h') libraries = ['util'] + eci = ExternalCompilationInfo( includes=includes, libraries=libraries, ) +def external(name, args, result, compilation_info=eci, **kwds): + return rffi.llexternal(name, args, result, + compilation_info=compilation_info, **kwds) + + +if os.name == 'nt': + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + rffi.INT, compilation_info=errno_eci, + )) + c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", + [], rffi.VOIDP, compilation_info=errno_eci)) + c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph", + [rffi.VOIDP], lltype.Void, + compilation_info=errno_eci)) + + @enforceargs(int) + def _validate_fd(fd): + if not is_valid_fd(fd): + from errno import EBADF + raise OSError(EBADF, 'Bad file descriptor') + + class FdValidator(object): + + def __init__(self, fd): + _validate_fd(fd) + + def __enter__(self): + self.invalid_param_hndlr = c_enter_suppress_iph() + return self + + def __exit__(self, *args): + c_exit_suppress_iph(self.invalid_param_hndlr) + + def _bound_for_write(fd, count): + if count > 32767 and c_isatty(fd): + # CPython Issue #11395, PyPy Issue #2636: the Windows console + # returns an error (12: not enough space error) on writing into + # stdout if stdout mode is binary and the length is greater than + # 66,000 bytes (or less, depending on heap usage). Can't easily + # test that, because we need 'fd' to be non-redirected... + count = 32767 + elif count > 0x7fffffff: + count = 0x7fffffff + return count +else: + class FdValidator(object): + + def __init__(self, fd): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def _bound_for_write(fd, count): + return count + +def closerange(fd_low, fd_high): + # this behaves like os.closerange() from Python 2.6. + for fd in xrange(fd_low, fd_high): + try: + with FdValidator(fd): + os.close(fd) + except OSError: + pass + class CConfig: _compilation_info_ = eci SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET') @@ -323,10 +381,6 @@ config = rffi_platform.configure(CConfig) globals().update(config) -def external(name, args, result, compilation_info=eci, **kwds): - return rffi.llexternal(name, args, result, - compilation_info=compilation_info, **kwds) - # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. @@ -418,12 +472,12 @@ return result def _dup(fd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup(fd) - else: - res = c_dup_noninheritable(fd) - return res + with FdValidator(fd): + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res @replace_os_function('dup') def dup(fd, inheritable=True): @@ -432,12 +486,12 @@ @replace_os_function('dup2') def dup2(fd, newfd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup2(fd, newfd) - else: - res = c_dup2_noninheritable(fd, newfd) - handle_posix_error('dup2', res) + with FdValidator(fd): + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -465,25 +519,26 @@ def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - got = handle_posix_error('read', c_read(fd, void_buf, count)) - return buf.str(got) + with FdValidator(fd): + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + got = handle_posix_error('read', c_read(fd, void_buf, count)) + return buf.str(got) @replace_os_function('write') @enforceargs(int, None) def write(fd, data): count = len(data) - validate_fd(fd) - count = _bound_for_write(fd, count) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('write', c_write(fd, buf, count)) + with FdValidator(fd): + count = _bound_for_write(fd, count) + with rffi.scoped_nonmovingbuffer(data) as buf: + ret = c_write(fd, buf, count) + return handle_posix_error('write', ret) @replace_os_function('close') def close(fd): - validate_fd(fd) - handle_posix_error('close', c_close(fd)) + with FdValidator(fd): + handle_posix_error('close', c_close(fd)) c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, @@ -491,15 +546,15 @@ @replace_os_function('lseek') def lseek(fd, pos, how): - validate_fd(fd) - if SEEK_SET is not None: - if how == 0: - how = SEEK_SET - elif how == 1: - how = SEEK_CUR - elif how == 2: - how = SEEK_END - return handle_posix_error('lseek', c_lseek(fd, pos, how)) + with FdValidator(fd): + if SEEK_SET is not None: + if how == 0: + how = SEEK_SET + elif how == 1: + how = SEEK_CUR + elif how == 2: + how = SEEK_END + return handle_posix_error('lseek', c_lseek(fd, pos, how)) if not _WIN32: c_pread = external('pread', @@ -513,7 +568,6 @@ def pread(fd, count, offset): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) @@ -521,7 +575,6 @@ @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) - validate_fd(fd) with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) @@ -532,7 +585,6 @@ @enforceargs(int, None, None) def posix_fallocate(fd, offset, length): - validate_fd(fd) return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length)) if HAVE_FADVISE: @@ -554,7 +606,6 @@ @enforceargs(int, None, None, int) def posix_fadvise(fd, offset, length, advice): - validate_fd(fd) error = c_posix_fadvise(fd, offset, length, advice) error = widen(error) if error != 0: @@ -565,7 +616,6 @@ save_err=rffi.RFFI_SAVE_ERRNO) @enforceargs(int, None, None) def lockf(fd, cmd, length): - validate_fd(fd) return handle_posix_error('lockf', c_lockf(fd, cmd, length)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, @@ -579,18 +629,18 @@ @replace_os_function('ftruncate') def ftruncate(fd, length): - validate_fd(fd) - handle_posix_error('ftruncate', c_ftruncate(fd, length)) + with FdValidator(fd): + handle_posix_error('ftruncate', c_ftruncate(fd, length)) @replace_os_function('fsync') def fsync(fd): - validate_fd(fd) - handle_posix_error('fsync', c_fsync(fd)) + with FdValidator(fd): + handle_posix_error('fsync', c_fsync(fd)) @replace_os_function('fdatasync') def fdatasync(fd): - validate_fd(fd) - handle_posix_error('fdatasync', c_fdatasync(fd)) + with FdValidator(fd): + handle_posix_error('fdatasync', c_fdatasync(fd)) def sync(): c_sync() @@ -657,8 +707,8 @@ @replace_os_function('fchdir') def fchdir(fd): - validate_fd(fd) - handle_posix_error('fchdir', c_fchdir(fd)) + with FdValidator(fd): + handle_posix_error('fchdir', c_fchdir(fd)) @replace_os_function('access') @specialize.argtype(0) @@ -1098,9 +1148,10 @@ @replace_os_function('isatty') def isatty(fd): - if not is_valid_fd(fd): - return False - return c_isatty(fd) != 0 + print 'isatty' + with FdValidator(fd): + return c_isatty(fd) != 0 + return False c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -218,9 +218,9 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) def get_osfhandle(fd): - from rpython.rlib.rposix import validate_fd - validate_fd(fd) - handle = _get_osfhandle(fd) + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + handle = _get_osfhandle(fd) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -15,19 +15,6 @@ py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) -def test_get_osfhandle_raising(): - #try to test what kind of exception get_osfhandle raises w/out fd validation - py.test.skip('Crashes python') - fid = open(str(udir.join('validate_test.txt')), 'w') - fd = fid.fileno() - fid.close() - def validate_fd(fd): - return 1 - _validate_fd = rwin32.validate_fd - rwin32.validate_fd = validate_fd - raises(WindowsError, rwin32.get_osfhandle, fd) - rwin32.validate_fd = _validate_fd - def test_open_process(): pid = rwin32.GetCurrentProcessId() assert pid != 0 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -427,10 +427,14 @@ if len(headers_to_precompile)>0 and self.version >= 80: # at least from VS2013 onwards we need to include PCH # objects in the final link command - linkobjs = 'stdafx.obj @<<\n$(OBJECTS)\n<<' + linkobjs = 'stdafx.obj ' else: - linkobjs = '@<<\n$(OBJECTS)\n<<' - + linkobjs = '' + if len(' '.join(rel_ofiles)) > 2048: + # command line is limited in length, use a response file + linkobjs += '@<<\n$(OBJECTS)\n<<' + else: + linkobjs += '$(OBJECTS)' extra_deps = [] if icon and not shared: extra_deps.append('icon.res') From pypy.commits at gmail.com Tue Feb 6 14:05:59 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 11:05:59 -0800 (PST) Subject: [pypy-commit] pypy msvc14: global variables no longer exist, use functions instead Message-ID: <5a79fc97.cf0e1c0a.e86ee.148a@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93776:4c1dd22959c2 Date: 2018-02-06 19:59 +0100 http://bitbucket.org/pypy/pypy/changeset/4c1dd22959c2/ Log: global variables no longer exist, use functions instead diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -151,9 +151,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -187,20 +188,27 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } + long pypy_get_timezone() { long timezone; _get_timezone(&timezone); return timezone;}; + int pypy_get_daylight() { int daylight; _get_daylight(&daylight); return daylight;}; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + return 0; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -221,8 +229,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: From pypy.commits at gmail.com Tue Feb 6 22:08:20 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 19:08:20 -0800 (PST) Subject: [pypy-commit] pypy msvc14: refactor choosing msvc version Message-ID: <5a7a6da4.2696df0a.a82a5.827c@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93777:6806803cf056 Date: 2018-02-06 17:25 -0500 http://bitbucket.org/pypy/pypy/changeset/6806803cf056/ Log: refactor choosing msvc version diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -9,22 +9,36 @@ import rpython rpydir = str(py.path.local(rpython.__file__).join('..')) -def _get_compiler_type(cc, x64_flag): +def _get_compiler_type(cc, x64_flag, ver0=None): if not cc: cc = os.environ.get('CC','') if not cc: - return MsvcPlatform(x64=x64_flag) + return MsvcPlatform(x64=x64_flag, ver0=ver0) elif cc.startswith('mingw') or cc == 'gcc': return MingwPlatform(cc) - return MsvcPlatform(cc=cc, x64=x64_flag) + return MsvcPlatform(cc=cc, x64=x64_flag, ver0=ver0) -def Windows(cc=None): - return _get_compiler_type(cc, False) +def _get_vcver0(): + # try to get the compiler which served to compile python + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = int(sys.version[msc_pos+6:msc_pos+10]) + # 1500 -> 90, 1900 -> 140 + vsver = (msc_ver / 10) - 60 + return vsver + return None -def Windows_x64(cc=None): +def Windows(cc=None, ver0=None): + if ver0 is None: + ver0 = _get_vcver0() + return _get_compiler_type(cc, False, ver0=ver0) + +def Windows_x64(cc=None, ver0=None): raise Exception("Win64 is not supported. You must either build for Win32" " or contribute the missing support in PyPy.") - return _get_compiler_type(cc, True) + if ver0 is None: + ver0 = _get_vcver0() + return _get_compiler_type(cc, True, ver0=ver0) def _find_vcvarsall(version, x64flag): import rpython.tool.setuptools_msvc as msvc @@ -88,15 +102,10 @@ log.msg("Updated environment with vsver %d, using x64 %s" % (vsver, x64flag,)) return env -def find_msvc_env(x64flag=False): +def find_msvc_env(x64flag=False, ver0=None): vcvers = [140, 90, 100] - # First, try to get the compiler which served to compile python - msc_pos = sys.version.find('MSC v.') - if msc_pos != -1: - msc_ver = int(sys.version[msc_pos+6:msc_pos+10]) - # 1500 -> 90, 1900 -> 140 - vsver = (msc_ver / 10) - 60 - #vcvers.insert(0, vsver) + if ver0 in vcvers: + vcvers.insert(0, ver0) errs = [] for vsver in vcvers: env = _get_msvc_env(vsver, x64flag) @@ -148,10 +157,10 @@ shared_only = () environ = None - def __init__(self, cc=None, x64=False): + def __init__(self, cc=None, x64=False, ver0=None): self.x64 = x64 if cc is None: - msvc_compiler_environ, self.vsver = find_msvc_env(x64) + msvc_compiler_environ, self.vsver = find_msvc_env(x64, ver0=ver0) Platform.__init__(self, 'cl.exe') if msvc_compiler_environ: if x64: From pypy.commits at gmail.com Tue Feb 6 22:08:22 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 19:08:22 -0800 (PST) Subject: [pypy-commit] pypy msvc14: do not use precompiled headers for main.c, wmain.c Message-ID: <5a7a6da6.46b51c0a.e925c.081b@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93778:d3ca334e5537 Date: 2018-02-06 17:26 -0500 http://bitbucket.org/pypy/pypy/changeset/d3ca334e5537/ Log: do not use precompiled headers for main.c, wmain.c diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -380,6 +380,8 @@ ] if len(headers_to_precompile)>0: + if shared: + no_precompile_cfiles += ['main.c', 'wmain.c'] stdafx_h = path.join('stdafx.h') txt = '#ifndef PYPY_STDAFX_H\n' txt += '#define PYPY_STDAFX_H\n' From pypy.commits at gmail.com Tue Feb 6 22:08:24 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 06 Feb 2018 19:08:24 -0800 (PST) Subject: [pypy-commit] pypy msvc14: cleanup for non-msvc14 Message-ID: <5a7a6da8.46061c0a.68438.8706@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93779:80c21230ed5e Date: 2018-02-06 17:27 -0500 http://bitbucket.org/pypy/pypy/changeset/80c21230ed5e/ Log: cleanup for non-msvc14 diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -192,15 +192,22 @@ "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { long timezone; _get_timezone(&timezone); return timezone;}; - int pypy_get_daylight() { int daylight; _get_daylight(&daylight); return daylight;}; - int pypy_get_tzname(size_t len, int index, char * tzname) { - size_t s; - return 0; - errno_t ret = _get_tzname(&s, tzname, len, index); - return (int)s; - }; - void pypy__tzset() { _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -44,7 +44,7 @@ includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' /* Lifted completely from CPython 3 Modules/posixmodule.c */ - #if defined _MSC_VER && MSC_VER >= 1400 && _MSC_VER < 1900 + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { intptr_t osfhnd; @@ -96,8 +96,8 @@ errno = EBADF; return 0; } - RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL}; - RPY_EXTERN void exit_suppress_iph(void*) {}; + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; #elif defined _MSC_VER RPY_EXTERN int _PyVerify_fd(int fd) { @@ -133,8 +133,8 @@ { return 1; } - void* enter_suppress_iph(void) {return (void*) NULL}; - void exit_suppress_iph(void*) {}; + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] @@ -1148,7 +1148,6 @@ @replace_os_function('isatty') def isatty(fd): - print 'isatty' with FdValidator(fd): return c_isatty(fd) != 0 return False From pypy.commits at gmail.com Wed Feb 7 08:48:43 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Feb 2018 05:48:43 -0800 (PST) Subject: [pypy-commit] pypy default: fix test Message-ID: <5a7b03bb.23b8df0a.a3b93.519e@mx.google.com> Author: Matti Picus Branch: Changeset: r93780:14abb2e081c8 Date: 2018-02-07 15:47 +0200 http://bitbucket.org/pypy/pypy/changeset/14abb2e081c8/ Log: fix test diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -244,7 +244,7 @@ if not cls.runappdirect: cls.total_mem = 0 - def add_memory_pressure(estimate): + def add_memory_pressure(estimate, object=None): assert estimate >= 0 cls.total_mem += estimate cls.orig_add_memory_pressure = [rgc.add_memory_pressure] From pypy.commits at gmail.com Wed Feb 7 08:53:22 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Feb 2018 05:53:22 -0800 (PST) Subject: [pypy-commit] pypy msvc14: merge default into branch Message-ID: <5a7b04d2.d3afdf0a.dfae6.9e42@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93781:d284e7798f85 Date: 2018-02-07 15:52 +0200 http://bitbucket.org/pypy/pypy/changeset/d284e7798f85/ Log: merge default into branch diff too long, truncating to 2000 out of 2727 lines diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.3 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,37 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1415,9 +1415,14 @@ self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = dateinterop.__new__(cls) self._year = year @@ -1491,7 +1496,7 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): @@ -1800,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1808,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,29 @@ .. branch: cpyext-datetime2 Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1096,17 +1096,21 @@ encoding = self.compile_info.encoding flags = self.compile_info.flags unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - try: - sub_strings_w = [parsestring.parsestr(space, encoding, atom_node.get_child(i).get_value(), - unicode_literals) - for i in range(atom_node.num_children())] - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise self.error('(unicode error) %s' % errmsg, atom_node) + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), + unicode_literals)) + except error.OperationError as e: + if not e.match(space, space.w_UnicodeError): + raise + # UnicodeError in literal: turn into SyntaxError + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + if child is None: + child = atom_node + raise self.error('(unicode error) %s' % errmsg, child) # This implements implicit string concatenation. if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1250,3 +1250,11 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -208,6 +208,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -134,7 +134,7 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.3" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -61,7 +61,8 @@ ctx = ropenssl.EVP_MD_CTX_new() if ctx is None: raise MemoryError - rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size, + self) try: if copy_from: if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from): diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1316,8 +1316,9 @@ if not ctx: raise ssl_error(space, "failed to allocate SSL context") - rgc.add_memory_pressure(10 * 1024 * 1024) self = space.allocate_instance(_SSLContext, w_subtype) + assert isinstance(self, _SSLContext) + rgc.add_memory_pressure(10 * 1024 * 1024, self) self.ctx = ctx self.check_hostname = False self.register_finalizer(space) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -267,7 +267,7 @@ raise oefmt(space.w_ValueError, "sequence.index(x): x not in sequence") class CPyListStrategy(ListStrategy): - erase, unerase = rerased.new_erasing_pair("empty") + erase, unerase = rerased.new_erasing_pair("cpylist") erase = staticmethod(erase) unerase = staticmethod(unerase) diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -244,7 +244,7 @@ if not cls.runappdirect: cls.total_mem = 0 - def add_memory_pressure(estimate): + def add_memory_pressure(estimate, object=None): assert estimate >= 0 cls.total_mem += estimate cls.orig_add_memory_pressure = [rgc.add_memory_pressure] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,10 +3,20 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -157,6 +167,25 @@ assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + def test_shortcut(self): + # test that instances of classes that are defined in C become an + # instance of W_BaseCPyObject and thus can be converted faster back to + # their pyobj, because they store a pointer to it directly. + if self.runappdirect: + skip("can't run with -A") + module = self.import_module(name='foo') + obj = module.fooType() + assert self._check_uses_shortcut(obj) + # W_TypeObjects use shortcut + assert self._check_uses_shortcut(object) + assert self._check_uses_shortcut(type) + # None, True, False use shortcut + assert self._check_uses_shortcut(None) + assert self._check_uses_shortcut(True) + assert self._check_uses_shortcut(False) + assert not self._check_uses_shortcut(1) + assert not self._check_uses_shortcut(object()) + def test_multiple_inheritance1(self): module = self.import_module(name='foo') obj = module.UnicodeSubtype(u'xyz') @@ -259,6 +288,11 @@ cmpr = module.OldCmpType() assert cmpr < cmpr + def test_unhashable_when_tpcompare(self): + module = self.import_module("comparisons") + cmpr = module.OldCmpType() + raises(TypeError, hash, cmpr) + def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -340,8 +340,12 @@ if len(slot_names) == 1: func = getattr(pto, slot_names[0]) if slot_names[0] == 'c_tp_hash': - if hash_not_impl == func: - # special case for tp_hash == PyObject_HashNotImplemented + # two special cases where __hash__ is explicitly set to None + # (which leads to an unhashable type): + # 1) tp_hash == PyObject_HashNotImplemented + # 2) tp_hash == NULL and either of tp_compare or tp_richcompare are not NULL + if hash_not_impl == func or ( + not func and (pto.c_tp_compare or pto.c_tp_richcompare)): dict_w[method_name] = space.w_None continue else: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -19,6 +19,7 @@ space.config.translation.gctransformer == "framework"): self.appleveldefs.update({ 'dump_rpy_heap': 'app_referents.dump_rpy_heap', + 'get_stats': 'app_referents.get_stats', }) self.interpleveldefs.update({ 'get_rpy_roots': 'referents.get_rpy_roots', @@ -28,6 +29,7 @@ 'get_objects': 'referents.get_objects', 'get_referents': 'referents.get_referents', 'get_referrers': 'referents.get_referrers', + '_get_stats': 'referents.get_stats', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -48,3 +48,66 @@ file.flush() fd = file.fileno() gc._dump_rpy_heap(fd) + +class GcStats(object): + def __init__(self, s): + self._s = s + for item in ('total_gc_memory', 'jit_backend_used', + 'total_memory_pressure', + 'total_allocated_memory', 'jit_backend_allocated', + 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', + 'total_rawmalloced_memory', 'nursery_size', + 'peak_arena_memory', 'peak_rawmalloced_memory'): + setattr(self, item, self._format(getattr(self._s, item))) + self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + + self._s.jit_backend_used) + self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + + self._s.jit_backend_allocated) + + def _format(self, v): + if v < 1000000: + # bit unlikely ;-) + return "%.1fkB" % (v / 1024.) + return "%.1fMB" % (v / 1024. / 1024.) + + def __repr__(self): + if self._s.total_memory_pressure != -1: + extra = "\nmemory pressure: %s" % self.total_memory_pressure + else: + extra = "" + return """Total memory consumed: + GC used: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler used: %s%s + ----------------------------- + Total: %s + + Total memory allocated: + GC allocated: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler allocated: %s%s + ----------------------------- + Total: %s + """ % (self.total_gc_memory, self.peak_memory, + self.total_arena_memory, + self.total_rawmalloced_memory, + self.nursery_size, + self.jit_backend_used, + extra, + self.memory_used_sum, + + self.total_allocated_memory, self.peak_allocated_memory, + self.peak_arena_memory, + self.peak_rawmalloced_memory, + self.nursery_size, + self.jit_backend_allocated, + extra, + self.memory_allocated_sum) + + +def get_stats(): + return GcStats(gc._get_stats()) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -1,7 +1,7 @@ -from rpython.rlib import rgc +from rpython.rlib import rgc, jit_hooks from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.error import oefmt, wrap_oserror from rpython.rlib.objectmodel import we_are_translated @@ -170,3 +170,53 @@ l = rgc.get_typeids_list() list_w = [space.newint(l[i]) for i in range(len(l))] return space.newlist(list_w) + +class W_GcStats(W_Root): + def __init__(self, memory_pressure): + if memory_pressure: + self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + else: + self.total_memory_pressure = -1 + self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY) + self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY) + self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY) + self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY) + self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None) + self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None) + self.total_arena_memory = rgc.get_stats(rgc.TOTAL_ARENA_MEMORY) + self.total_rawmalloced_memory = rgc.get_stats( + rgc.TOTAL_RAWMALLOCED_MEMORY) + self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) + self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) + self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + +W_GcStats.typedef = TypeDef("GcStats", + total_memory_pressure=interp_attrproperty("total_memory_pressure", + cls=W_GcStats, wrapfn="newint"), + total_gc_memory=interp_attrproperty("total_gc_memory", + cls=W_GcStats, wrapfn="newint"), + peak_allocated_memory=interp_attrproperty("peak_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + peak_memory=interp_attrproperty("peak_memory", + cls=W_GcStats, wrapfn="newint"), + total_allocated_memory=interp_attrproperty("total_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + jit_backend_allocated=interp_attrproperty("jit_backend_allocated", + cls=W_GcStats, wrapfn="newint"), + jit_backend_used=interp_attrproperty("jit_backend_used", + cls=W_GcStats, wrapfn="newint"), + total_arena_memory=interp_attrproperty("total_arena_memory", + cls=W_GcStats, wrapfn="newint"), + total_rawmalloced_memory=interp_attrproperty("total_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + peak_arena_memory=interp_attrproperty("peak_arena_memory", + cls=W_GcStats, wrapfn="newint"), + peak_rawmalloced_memory=interp_attrproperty("peak_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + nursery_size=interp_attrproperty("nursery_size", + cls=W_GcStats, wrapfn="newint"), +) + + at unwrap_spec(memory_pressure=bool) +def get_stats(space, memory_pressure=False): + return W_GcStats(memory_pressure) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -840,11 +840,11 @@ # Currently this is just the size of the pointer and some estimated bytes. # The struct isn't actually defined in expat.h - it is in xmlparse.c # XXX: find a good estimate of the XML_ParserStruct - rgc.add_memory_pressure(XML_Parser_SIZE + 300) if not xmlparser: raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed") parser = W_XMLParserType(space, xmlparser, w_intern) + rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser) XML_SetUnknownEncodingHandler( parser.itself, UnknownEncodingHandlerData_callback, rffi.cast(rffi.VOIDP, parser.id)) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2298,3 +2298,11 @@ else: assert lib.__loader__ is None assert lib.__spec__ is None + +def test_realize_struct_error(): + ffi = FFI() + ffi.cdef("""typedef ... foo_t; struct foo_s { void (*x)(foo_t); };""") + lib = verify(ffi, "test_realize_struct_error", """ + typedef int foo_t; struct foo_s { void (*x)(foo_t); }; + """) + py.test.raises(TypeError, ffi.new, "struct foo_s *") diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -7,7 +7,7 @@ from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, - W_DictObject, + W_DictObject, BytesDictStrategy, UnicodeDictStrategy ) from pypy.objspace.std.typeobject import MutableCell @@ -25,6 +25,10 @@ # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative +# the maximum number of attributes stored in mapdict (afterwards just use a +# dict) +LIMIT_MAP_ATTRIBUTES = 80 + class AbstractAttribute(object): _immutable_fields_ = ['terminator'] @@ -252,6 +256,9 @@ def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") + def materialize_str_dict(self, space, obj, str_dict): + raise NotImplementedError("abstract base class") + def remove_dict_entries(self, obj): raise NotImplementedError("abstract base class") @@ -271,6 +278,13 @@ def _write_terminator(self, obj, name, index, w_value): obj._get_mapdict_map().add_attr(obj, name, index, w_value) + if index == DICT and obj._get_mapdict_map().length() >= LIMIT_MAP_ATTRIBUTES: + space = self.space + w_dict = obj.getdict(space) + assert isinstance(w_dict, W_DictMultiObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, MapDictStrategy) + strategy.switch_to_text_strategy(w_dict) return True def copy(self, obj): @@ -301,6 +315,12 @@ self.devolved_dict_terminator = DevolvedDictTerminator(space, w_cls) def materialize_r_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def materialize_str_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def _make_devolved(self, space): result = Object() result.space = space result._mapdict_init_empty(self.devolved_dict_terminator) @@ -407,6 +427,14 @@ self._copy_attr(obj, new_obj) return new_obj + def materialize_str_dict(self, space, obj, str_dict): + new_obj = self.back.materialize_str_dict(space, obj, str_dict) + if self.index == DICT: + str_dict[self.name] = obj._mapdict_read_storage(self.storageindex) + else: + self._copy_attr(obj, new_obj) + return new_obj + def remove_dict_entries(self, obj): new_obj = self.back.remove_dict_entries(obj) if self.index != DICT: @@ -738,6 +766,15 @@ assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) + def switch_to_text_strategy(self, w_dict): + w_obj = self.unerase(w_dict.dstorage) + strategy = self.space.fromcache(BytesDictStrategy) + str_dict = strategy.unerase(strategy.get_empty_storage()) + w_dict.set_strategy(strategy) + w_dict.dstorage = strategy.erase(str_dict) + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None + materialize_str_dict(self.space, w_obj, str_dict) + def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) @@ -833,6 +870,11 @@ new_obj = map.materialize_r_dict(space, obj, dict_w) obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) +def materialize_str_dict(space, obj, dict_w): + map = obj._get_mapdict_map() + new_obj = map.materialize_str_dict(space, obj, dict_w) + obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) + class IteratorMixin(object): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -111,6 +111,33 @@ assert obj2.getdictvalue(space, "b") == 60 assert obj2.map is obj.map +def test_add_attribute_limit(): + for numslots in [0, 10, 100]: + cls = Class() + obj = cls.instantiate() + for i in range(numslots): + obj.setslotvalue(i, i) # some extra slots too, sometimes + # test that eventually attributes are really just stored in a dictionary + for i in range(1000): + obj.setdictvalue(space, str(i), i) + # moved to dict (which is the remaining non-slot item) + assert len(obj.storage) == 1 + numslots + + for i in range(1000): + assert obj.getdictvalue(space, str(i)) == i + for i in range(numslots): + assert obj.getslotvalue(i) == i # check extra slots + + # this doesn't happen with slots + cls = Class() + obj = cls.instantiate() + for i in range(1000): + obj.setslotvalue(i, i) + assert len(obj.storage) == 1000 + + for i in range(1000): + assert obj.getslotvalue(i) == i + def test_insert_different_orders(): cls = Class() obj = cls.instantiate() diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -71,6 +71,7 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() + self.memory_pressure_types = set() self.register_builtins() diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -675,6 +675,8 @@ def _ll_1_gc_add_memory_pressure(num): llop.gc_add_memory_pressure(lltype.Void, num) + def _ll_2_gc_add_memory_pressure(num, obj): + llop.gc_add_memory_pressure(lltype.Void, num, obj) def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None): diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -83,7 +83,9 @@ has_custom_trace, fast_path_tracing, has_gcptr, - cannot_pin): + cannot_pin, + has_memory_pressure, + get_memory_pressure_ofs): self.finalizer_handlers = finalizer_handlers self.destructor_or_custom_trace = destructor_or_custom_trace self.is_old_style_finalizer = is_old_style_finalizer @@ -103,6 +105,8 @@ self.fast_path_tracing = fast_path_tracing self.has_gcptr = has_gcptr self.cannot_pin = cannot_pin + self.has_memory_pressure = has_memory_pressure + self.get_memory_pressure_ofs = get_memory_pressure_ofs def get_member_index(self, type_id): return self.member_index(type_id) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -72,6 +72,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize +from rpython.rlib import rgc from rpython.memory.gc.minimarkpage import out_of_memory # @@ -371,6 +372,7 @@ self.old_rawmalloced_objects = self.AddressStack() self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) + self.rawmalloced_peak_size = r_uint(0) self.gc_state = STATE_SCANNING # @@ -996,6 +998,8 @@ # Record the newly allocated object and its full malloced size. # The object is young or old depending on the argument. self.rawmalloced_total_size += r_uint(allocsize) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) if alloc_young: if not self.young_rawmalloced_objects: self.young_rawmalloced_objects = self.AddressDict() @@ -1023,7 +1027,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): # Decrement by 'sizehint' plus a very little bit extra. This # is needed e.g. for _rawffi, which may allocate a lot of tiny # arrays. @@ -1183,6 +1187,24 @@ """ return self.ac.total_memory_used + self.rawmalloced_total_size + def get_total_memory_alloced(self): + """ Return the total memory allocated + """ + return self.ac.total_memory_alloced + self.rawmalloced_total_size + + def get_peak_memory_alloced(self): + """ Return the peak memory ever allocated. The peaks + can be at different times, but we just don't worry for now + """ + return self.ac.peak_memory_alloced + self.rawmalloced_peak_size + + def get_peak_memory_used(self): + """ Return the peak memory GC felt ever responsible for + """ + mem_allocated = max(self.ac.peak_memory_used, + self.ac.total_memory_used) + return mem_allocated + self.rawmalloced_peak_size + def threshold_reached(self, extra=0): return (self.next_major_collection_threshold - float(self.get_total_memory_used())) < float(extra) @@ -2155,6 +2177,8 @@ # size_gc_header = self.gcheaderbuilder.size_gc_header self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize)) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) self.old_rawmalloced_objects.append(arena + size_gc_header) return arena @@ -2932,6 +2956,32 @@ self.old_objects_with_weakrefs.delete() self.old_objects_with_weakrefs = new_with_weakref + def get_stats(self, stats_no): + from rpython.memory.gc import inspector + + if stats_no == rgc.TOTAL_MEMORY: + return intmask(self.get_total_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_MEMORY: + return intmask(self.get_peak_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_ALLOCATED_MEMORY: + return intmask(self.get_peak_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY: + return intmask(self.get_total_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_MEMORY_PRESSURE: + return inspector.count_memory_pressure(self) + elif stats_no == rgc.TOTAL_ARENA_MEMORY: + return intmask(self.ac.total_memory_used) + elif stats_no == rgc.TOTAL_RAWMALLOCED_MEMORY: + return intmask(self.rawmalloced_total_size) + elif stats_no == rgc.PEAK_RAWMALLOCED_MEMORY: + return intmask(self.rawmalloced_peak_size) + elif stats_no == rgc.PEAK_ARENA_MEMORY: + return intmask(max(self.ac.peak_memory_used, + self.ac.total_memory_used)) + elif stats_no == rgc.NURSERY_SIZE: + return intmask(self.nursery_size) + return 0 + # ---------- # RawRefCount diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py --- a/rpython/memory/gc/inspector.py +++ b/rpython/memory/gc/inspector.py @@ -2,6 +2,7 @@ Utility RPython functions to inspect objects in the GC. """ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.objectmodel import free_non_gc_object from rpython.rlib import rposix, rgc, jit @@ -92,17 +93,12 @@ AddressStack = get_address_stack() -class HeapDumper(object): - _alloc_flavor_ = "raw" - BUFSIZE = 8192 # words +class BaseWalker(object): + _alloc_flavor_ = 'raw' - def __init__(self, gc, fd): + def __init__(self, gc): self.gc = gc self.gcflag = gc.gcflag_extra - self.fd = rffi.cast(rffi.INT, fd) - self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, - flavor='raw') - self.buf_count = 0 if self.gcflag == 0: self.seen = AddressDict() self.pending = AddressStack() @@ -111,8 +107,107 @@ if self.gcflag == 0: self.seen.delete() self.pending.delete() + free_non_gc_object(self) + + def add_roots(self): + self.gc.enumerate_all_roots(_hd_add_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.walk(pendingroots) + pendingroots.delete() + self.end_add_roots_marker() + + def end_add_roots_marker(self): + pass + + def add(self, obj): + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) + + def walk(self, pending): + while pending.non_empty(): + self.processobj(pending.pop()) + + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unobj(self, obj): + gc = self.gc + gc.trace(obj, self._unref, None) + + def _unref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unobj(pending.pop()) + + def finish_processing(self): + if self.gcflag != 0: + self.clear_gcflag_again() + self.unwalk(self.pending) + + def process(self): + self.add_roots() + self.walk(self.pending) + + +class MemoryPressureCounter(BaseWalker): + + def __init__(self, gc): + self.count = 0 + BaseWalker.__init__(self, gc) + + def processobj(self, obj): + gc = self.gc + typeid = gc.get_type_id(obj) + if gc.has_memory_pressure(typeid): + ofs = gc.get_memory_pressure_ofs(typeid) + val = (obj + ofs).signed[0] + self.count += val + gc.trace(obj, self._ref, None) + + def _ref(self, pointer, _): + obj = pointer.address[0] + self.add(obj) + + +class HeapDumper(BaseWalker): + BUFSIZE = 8192 # words + + def __init__(self, gc, fd): + BaseWalker.__init__(self, gc) + self.fd = rffi.cast(rffi.INT, fd) + self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, + flavor='raw') + self.buf_count = 0 + + def delete(self): lltype.free(self.writebuffer, flavor='raw') - free_non_gc_object(self) + BaseWalker.delete(self) @jit.dont_look_inside def flush(self): @@ -143,6 +238,7 @@ self.write(0) self.write(0) self.write(-1) + end_add_roots_marker = write_marker def writeobj(self, obj): gc = self.gc @@ -152,64 +248,13 @@ self.write(gc.get_size_incl_hash(obj)) gc.trace(obj, self._writeref, None) self.write(-1) + processobj = writeobj def _writeref(self, pointer, _): obj = pointer.address[0] self.write(llmemory.cast_adr_to_int(obj)) self.add(obj) - def add(self, obj): - if self.gcflag == 0: - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) - else: - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) == 0: - hdr.tid |= self.gcflag - self.pending.append(obj) - - def add_roots(self): - self.gc.enumerate_all_roots(_hd_add_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.walk(pendingroots) - pendingroots.delete() - self.write_marker() - - def walk(self, pending): - while pending.non_empty(): - self.writeobj(pending.pop()) - - # ---------- - # A simplified copy of the above, to make sure we walk again all the - # objects to clear the 'gcflag'. - - def unwriteobj(self, obj): - gc = self.gc - gc.trace(obj, self._unwriteref, None) - - def _unwriteref(self, pointer, _): - obj = pointer.address[0] - self.unadd(obj) - - def unadd(self, obj): - assert self.gcflag != 0 - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) != 0: - hdr.tid &= ~self.gcflag - self.pending.append(obj) - - def clear_gcflag_again(self): - self.gc.enumerate_all_roots(_hd_unadd_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.unwalk(pendingroots) - pendingroots.delete() - - def unwalk(self, pending): - while pending.non_empty(): - self.unwriteobj(pending.pop()) def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) @@ -219,15 +264,20 @@ def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) - heapdumper.add_roots() - heapdumper.walk(heapdumper.pending) + heapdumper.process() heapdumper.flush() - if heapdumper.gcflag != 0: - heapdumper.clear_gcflag_again() - heapdumper.unwalk(heapdumper.pending) + heapdumper.finish_processing() heapdumper.delete() return True +def count_memory_pressure(gc): + counter = MemoryPressureCounter(gc) + counter.process() + counter.finish_processing() + res = counter.count + counter.delete() + return res + def get_typeids_z(gc): srcaddress = gc.root_walker.gcdata.typeids_z return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR)) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -828,7 +828,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): self.next_major_collection_threshold -= sizehint if self.next_major_collection_threshold < 0: # cannot trigger a full collection now, but we can ensure diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -141,6 +141,9 @@ # the total memory used, counting every block in use, without # the additional bookkeeping stuff. self.total_memory_used = r_uint(0) + self.peak_memory_used = r_uint(0) + self.total_memory_alloced = r_uint(0) + self.peak_memory_alloced = r_uint(0) def _new_page_ptr_list(self, length): @@ -293,7 +296,11 @@ # # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address - arena_base = llarena.arena_malloc(self.arena_size, False) + arena_base = llarena.arena_mmap(self.arena_size) + self.total_memory_alloced += self.arena_size + self.peak_memory_alloced = max(self.total_memory_alloced, + self.peak_memory_alloced) + if not arena_base: out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size @@ -321,6 +328,8 @@ """Prepare calls to mass_free_incremental(): moves the chained lists into 'self.old_xxx'. """ + self.peak_memory_used = max(self.peak_memory_used, + self.total_memory_used) self.total_memory_used = r_uint(0) # size_class = self.small_request_threshold >> WORD_POWER_2 @@ -397,8 +406,8 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. - llarena.arena_reset(arena.base, self.arena_size, 4) - llarena.arena_free(arena.base) + llarena.arena_munmap(arena.base, self.arena_size) + self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 # diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -392,23 +392,30 @@ inline = True) if getattr(GCClass, 'raw_malloc_memory_pressure', False): - def raw_malloc_memory_pressure_varsize(length, itemsize): + def raw_malloc_memory_pressure_varsize(length, itemsize, adr): totalmem = length * itemsize if totalmem > 0: - gcdata.gc.raw_malloc_memory_pressure(totalmem) + gcdata.gc.raw_malloc_memory_pressure(totalmem, adr) #else: probably an overflow -- the following rawmalloc # will fail then - def raw_malloc_memory_pressure(sizehint): - gcdata.gc.raw_malloc_memory_pressure(sizehint) + def raw_malloc_memory_pressure(sizehint, adr): + gcdata.gc.raw_malloc_memory_pressure(sizehint, adr) self.raw_malloc_memory_pressure_varsize_ptr = getfn( raw_malloc_memory_pressure_varsize, - [annmodel.SomeInteger(), annmodel.SomeInteger()], + [annmodel.SomeInteger(), annmodel.SomeInteger(), + SomeAddress()], annmodel.s_None, minimal_transform = False) self.raw_malloc_memory_pressure_ptr = getfn( raw_malloc_memory_pressure, - [annmodel.SomeInteger()], + [annmodel.SomeInteger(), SomeAddress()], annmodel.s_None, minimal_transform = False) + if getattr(GCClass, 'get_stats', False): + def get_stats(stats_no): + return gcdata.gc.get_stats(stats_no) + self.get_stats_ptr = getfn(get_stats, [annmodel.SomeInteger()], + annmodel.SomeInteger()) + self.identityhash_ptr = getfn(GCClass.identityhash.im_func, [s_gc, s_gcref], @@ -831,6 +838,39 @@ gct_fv_gc_malloc_varsize = gct_fv_gc_malloc + def gct_gc_add_memory_pressure(self, hop): + def _find_correct_type(TP): + T = TP.TO + while 'special_memory_pressure' not in T._flds: + T = T._flds['super'] + return T + + if hasattr(self, 'raw_malloc_memory_pressure_ptr'): + op = hop.spaceop + size = op.args[0] + if len(op.args) == 2: + v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure") + T = _find_correct_type(op.args[1].concretetype) + v_inst = hop.genop("cast_pointer", [op.args[1]], + resulttype=lltype.Ptr(T)) + hop.genop("bare_setfield", [v_inst, v_fld, size]) + v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]], + resulttype=llmemory.Address) + else: + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) + hop.genop("direct_call", [self.raw_malloc_memory_pressure_ptr, + size, v_adr]) + + + def gct_gc_get_stats(self, hop): + if hasattr(self, 'get_stats_ptr'): + return hop.genop("direct_call", + [self.get_stats_ptr, hop.spaceop.args[0]], + resultvar=hop.spaceop.result) + hop.genop("same_as", [rmodel.inputconst(lltype.Signed, 0)], + resultvar=hop.spaceop.result) + + def gct_gc__collect(self, hop): op = hop.spaceop if len(op.args) == 1: diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -535,12 +535,7 @@ return self.varsize_malloc_helper(hop, flags, meth, []) def gct_gc_add_memory_pressure(self, hop): - if hasattr(self, 'raw_malloc_memory_pressure_ptr'): - op = hop.spaceop - size = op.args[0] - return hop.genop("direct_call", - [self.raw_malloc_memory_pressure_ptr, - size]) + pass def varsize_malloc_helper(self, hop, flags, meth, extraargs): def intconst(c): return rmodel.inputconst(lltype.Signed, c) @@ -574,9 +569,10 @@ c_offset_to_length): if flags.get('add_memory_pressure', False): if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'): + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) hop.genop("direct_call", [self.raw_malloc_memory_pressure_varsize_ptr, - v_length, c_item_size]) + v_length, c_item_size, v_adr]) if c_offset_to_length is None: if flags.get('zero'): fnptr = self.raw_malloc_varsize_no_length_zero_ptr diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -21,13 +21,21 @@ # A destructor is called when the object is about to be freed. # A custom tracer (CT) enumerates the addresses that contain GCREFs. # Both are called with the address of the object as only argument. + # They're embedded in a struct that has raw_memory_offset as another + # argument, which is only valid if T_HAS_MEMORY_PRESSURE is set CUSTOM_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) CUSTOM_FUNC_PTR = lltype.Ptr(CUSTOM_FUNC) + CUSTOM_DATA_STRUCT = lltype.Struct('custom_data', + ('customfunc', CUSTOM_FUNC_PTR), + ('memory_pressure_offset', lltype.Signed), # offset to where the amount + # of owned memory pressure is stored + ) + CUSTOM_DATA_STRUCT_PTR = lltype.Ptr(CUSTOM_DATA_STRUCT) # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", ("infobits", lltype.Signed), # combination of the T_xxx consts - ("customfunc", CUSTOM_FUNC_PTR), + ("customdata", CUSTOM_DATA_STRUCT_PTR), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), hints={'immutable': True}, @@ -81,14 +89,14 @@ def q_cannot_pin(self, typeid): typeinfo = self.get(typeid) ANY = (T_HAS_GCPTR | T_IS_WEAKREF) - return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customfunc) + return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customdata) def q_finalizer_handlers(self): adr = self.finalizer_handlers # set from framework.py or gcwrapper.py return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY)) def q_destructor_or_custom_trace(self, typeid): - return self.get(typeid).customfunc + return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): typeinfo = self.get(typeid) @@ -139,6 +147,15 @@ infobits = self.get(typeid).infobits return infobits & T_ANY_SLOW_FLAG == 0 + def q_has_memory_pressure(self, typeid): + infobits = self.get(typeid).infobits + return infobits & T_HAS_MEMORY_PRESSURE != 0 + + def q_get_memory_pressure_ofs(self, typeid): + infobits = self.get(typeid).infobits + assert infobits & T_HAS_MEMORY_PRESSURE != 0 + return self.get(typeid).customdata.memory_pressure_offset + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -159,7 +176,9 @@ self.q_has_custom_trace, self.q_fast_path_tracing, self.q_has_gcptr, - self.q_cannot_pin) + self.q_cannot_pin, + self.q_has_memory_pressure, + self.q_get_memory_pressure_ofs) def _has_got_custom_trace(self, typeid): type_info = self.get(typeid) @@ -176,8 +195,9 @@ T_HAS_CUSTOM_TRACE = 0x200000 T_HAS_OLDSTYLE_FINALIZER = 0x400000 T_HAS_GCPTR = 0x1000000 -T_KEY_MASK = intmask(0xFE000000) # bug detection only -T_KEY_VALUE = intmask(0x5A000000) # bug detection only +T_HAS_MEMORY_PRESSURE = 0x2000000 # first field is memory pressure field +T_KEY_MASK = intmask(0xFC000000) # bug detection only +T_KEY_VALUE = intmask(0x58000000) # bug detection only def _check_valid_type_info(p): ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id") @@ -192,6 +212,25 @@ ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") +def has_special_memory_pressure(TYPE): + if TYPE._is_varsize(): + return False + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return True + if 'super' not in T._flds: + return False + T = T._flds['super'] + +def get_memory_pressure_ofs(TYPE): + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return llmemory.offsetof(T, 'special_memory_pressure') + if 'super' not in T._flds: + assert False, "get_ and has_memory_pressure disagree" + T = T._flds['super'] def encode_type_shape(builder, info, TYPE, index): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" @@ -202,12 +241,18 @@ infobits |= T_HAS_GCPTR # fptrs = builder.special_funcptr_for_type(TYPE) - if fptrs: + if fptrs or has_special_memory_pressure(TYPE): + customdata = lltype.malloc(GCData.CUSTOM_DATA_STRUCT, flavor='raw', + immortal=True) + info.customdata = customdata if "destructor" in fptrs: - info.customfunc = fptrs["destructor"] + customdata.customfunc = fptrs["destructor"] if "old_style_finalizer" in fptrs: - info.customfunc = fptrs["old_style_finalizer"] + customdata.customfunc = fptrs["old_style_finalizer"] infobits |= T_HAS_OLDSTYLE_FINALIZER + if has_special_memory_pressure(TYPE): + infobits |= T_HAS_MEMORY_PRESSURE + info.customdata.memory_pressure_offset = get_memory_pressure_ofs(TYPE) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -83,9 +83,9 @@ def gettypeid(self, obj): return self.get_type_id(lltype.typeOf(obj).TO) - def add_memory_pressure(self, size): + def add_memory_pressure(self, size, adr): if hasattr(self.gc, 'raw_malloc_memory_pressure'): - self.gc.raw_malloc_memory_pressure(size) + self.gc.raw_malloc_memory_pressure(size, adr) def shrink_array(self, p, smallersize): if hasattr(self.gc, 'shrink_array'): diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -695,9 +695,7 @@ return NULLRBIGINT if asize == 1: - if a._digits[0] == NULLDIGIT: - return NULLRBIGINT - elif a._digits[0] == ONEDIGIT: + if a._digits[0] == ONEDIGIT: return rbigint(b._digits[:b.size], a.sign * b.sign, b.size) elif bsize == 1: res = b.widedigit(0) * a.widedigit(0) @@ -784,35 +782,15 @@ @jit.elidable def mod(self, other): + if other.sign == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT - if other.sign != 0 and other.numdigits() == 1: - digit = other.digit(0) - if digit == 1: - return NULLRBIGINT - elif digit == 2: - modm = self.digit(0) & 1 - if modm: - return ONENEGATIVERBIGINT if other.sign == -1 else ONERBIGINT - return NULLRBIGINT - elif digit & (digit - 1) == 0: - mod = self.int_and_(digit - 1) - else: - # Perform - size = self.numdigits() - 1 - if size > 0: - rem = self.widedigit(size) - size -= 1 - while size >= 0: - rem = ((rem << SHIFT) + self.widedigit(size)) % digit - size -= 1 - else: - rem = self.digit(0) % digit - - if rem == 0: - return NULLRBIGINT - mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1) + if other.numdigits() == 1: + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_mod(otherint) else: div, mod = _divrem(self, other) if mod.sign * other.sign == -1: @@ -821,6 +799,8 @@ @jit.elidable def int_mod(self, other): + if other == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT @@ -828,7 +808,7 @@ # Fallback to long. return self.mod(rbigint.fromint(other)) - elif other != 0: + if 1: # preserve indentation to preserve history digit = abs(other) if digit == 1: return NULLRBIGINT @@ -842,6 +822,7 @@ else: # Perform size = self.numdigits() - 1 + if size > 0: rem = self.widedigit(size) size -= 1 @@ -854,8 +835,6 @@ if rem == 0: return NULLRBIGINT mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1) - else: - raise ZeroDivisionError("long division or modulo by zero") if mod.sign * (-1 if other < 0 else 1) == -1: mod = mod.int_add(other) @@ -1825,16 +1804,14 @@ ret._normalize() return ret -def _inplace_divrem1(pout, pin, n, size=0): +def _inplace_divrem1(pout, pin, n): """ Divide bigint pin by non-zero digit n, storing quotient in pout, and returning the remainder. It's OK for pin == pout on entry. """ rem = _widen_digit(0) assert n > 0 and n <= MASK - if not size: - size = pin.numdigits() - size -= 1 + size = pin.numdigits() - 1 while size >= 0: rem = (rem << SHIFT) | pin.widedigit(size) hi = rem // n @@ -2552,6 +2529,8 @@ maska ^= MASK maskb ^= MASK negz = -1 + else: + assert 0, "unreachable" # JRH: The original logic here was to allocate the result value (z) # as the longer of the two operands. However, there are some cases diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -598,21 +598,31 @@ return False return type(x).__module__ != '__builtin__' # keep non-builtins -def add_memory_pressure(estimate): +def add_memory_pressure(estimate, object=None): """Add memory pressure for OpaquePtrs.""" pass class AddMemoryPressureEntry(ExtRegistryEntry): _about_ = add_memory_pressure - def compute_result_annotation(self, s_nbytes): + def compute_result_annotation(self, s_nbytes, s_object=None): from rpython.annotator import model as annmodel + if s_object is not None: + if not isinstance(s_object, annmodel.SomeInstance): + raise Exception("Wrong kind of object passed to " + "add memory pressure") + self.bookkeeper.memory_pressure_types.add(s_object.classdef) return annmodel.s_None def specialize_call(self, hop): - [v_size] = hop.inputargs(lltype.Signed) + v_size = hop.inputarg(lltype.Signed, 0) + if len(hop.args_v) == 2: + v_obj = hop.inputarg(hop.args_r[1], 1) + args = [v_size, v_obj] + else: + args = [v_size] hop.exception_cannot_occur() - return hop.genop('gc_add_memory_pressure', [v_size], + return hop.genop('gc_add_memory_pressure', args, resulttype=lltype.Void) @@ -640,6 +650,17 @@ else: return id(gcref._x) +(TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, + PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, + TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, + NURSERY_SIZE) = range(10) + + at not_rpython +def get_stats(stat_no): + """ Long docstring goes here + """ + raise NotImplementedError + @not_rpython def dump_rpy_heap(fd): raise NotImplementedError @@ -834,6 +855,18 @@ return hop.genop('gc_get_rpy_type_index', vlist, resulttype = hop.r_result) +class Entry(ExtRegistryEntry): + _about_ = get_stats + def compute_result_annotation(self, s_no): + from rpython.annotator.model import SomeInteger + if not isinstance(s_no, SomeInteger): + raise Exception("expecting an integer") + return SomeInteger() + def specialize_call(self, hop): + args = hop.inputargs(lltype.Signed) + hop.exception_cannot_occur() + return hop.genop('gc_get_stats', args, resulttype=lltype.Signed) + @not_rpython def _is_rpy_instance(gcref): raise NotImplementedError diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -85,7 +85,11 @@ def allocate_lock(): - return Lock(allocate_ll_lock()) + # Add some memory pressure for the size of the lock because it is an + # Opaque object + lock = Lock(allocate_ll_lock()) + rgc.add_memory_pressure(TLOCKP_SIZE, lock) + return lock From pypy.commits at gmail.com Wed Feb 7 09:40:48 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Feb 2018 06:40:48 -0800 (PST) Subject: [pypy-commit] pypy msvc14: is_valid_fd no longer exists Message-ID: <5a7b0ff0.116e1c0a.36b2f.ceea@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93782:110e3c57792a Date: 2018-02-07 09:39 -0500 http://bitbucket.org/pypy/pypy/changeset/110e3c57792a/ Log: is_valid_fd no longer exists diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, FdValidator +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None From pypy.commits at gmail.com Wed Feb 7 15:38:02 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Feb 2018 12:38:02 -0800 (PST) Subject: [pypy-commit] pypy msvc14: merge default into branch Message-ID: <5a7b63aa.0abfdf0a.5734c.45c8@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93784:6268f664221b Date: 2018-02-07 15:37 -0500 http://bitbucket.org/pypy/pypy/changeset/6268f664221b/ Log: merge default into branch diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -9,6 +9,7 @@ WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT] assert 1 << WORD_POWER_2 == WORD +USE_MMAP = False # Terminology: the memory is subdivided into "arenas" containing "pages". # A page contains a number of allocated objects, called "blocks". @@ -296,7 +297,10 @@ # # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address - arena_base = llarena.arena_mmap(self.arena_size) + if USE_MMAP: + arena_base = llarena.arena_mmap(self.arena_size) + else: + arena_base = llarena.arena_malloc(self.arena_size, False) self.total_memory_alloced += self.arena_size self.peak_memory_alloced = max(self.total_memory_alloced, self.peak_memory_alloced) @@ -406,7 +410,11 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. - llarena.arena_munmap(arena.base, self.arena_size) + if USE_MMAP: + llarena.arena_munmap(arena.base, self.arena_size) + else: + llarena.arena_reset(arena.base, self.arena_size, 4) + llarena.arena_free(arena.base) self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import llmemory from rpython.rlib.rarithmetic import is_valid_int from rpython.rtyper.lltypesystem.lloperation import llop - +import os, sys # An "arena" is a large area of memory which can hold a number of # objects, not necessarily all of the same type or size. It's used by @@ -327,15 +327,16 @@ assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() -def arena_mmap(nbytes): - """Allocate and return a new arena, zero-initialized by the - system, calling mmap().""" - return arena_malloc(nbytes, True) +if os.name == 'posix': + def arena_mmap(nbytes): + """Allocate and return a new arena, zero-initialized by the + system, calling mmap().""" + return arena_malloc(nbytes, True) -def arena_munmap(arena_addr, nbytes): - """Release an arena allocated with arena_mmap().""" - arena_free(arena_addr) - assert nbytes == arena_addr.arena.nbytes + def arena_munmap(arena_addr, nbytes): + """Release an arena allocated with arena_mmap().""" + arena_free(arena_addr) + assert nbytes == arena_addr.arena.nbytes def arena_reset(arena_addr, size, zero): @@ -404,7 +405,6 @@ # We can tweak these implementations to be more suited to very large # chunks of memory. -import os, sys from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.extfunc import register_external from rpython.rtyper.tool.rffi_platform import memory_alignment @@ -541,30 +541,31 @@ llfakeimpl=arena_free, sandboxsafe=True) -def llimpl_arena_mmap(nbytes): - from rpython.rlib import rmmap - flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS - prot = rmmap.PROT_READ | rmmap.PROT_WRITE - p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( - lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) - if p == rffi.cast(llmemory.Address, -1): - p = rffi.cast(llmemory.Address, 0) - return p -register_external(arena_mmap, [int], llmemory.Address, - 'll_arena.arena_mmap', - llimpl=llimpl_arena_mmap, - llfakeimpl=arena_mmap, - sandboxsafe=True) +if os.name == 'posix': + def llimpl_arena_mmap(nbytes): + from rpython.rlib import rmmap + flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS + prot = rmmap.PROT_READ | rmmap.PROT_WRITE + p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( + lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) + if p == rffi.cast(llmemory.Address, -1): + p = rffi.cast(llmemory.Address, 0) + return p + register_external(arena_mmap, [int], llmemory.Address, + 'll_arena.arena_mmap', + llimpl=llimpl_arena_mmap, + llfakeimpl=arena_mmap, + sandboxsafe=True) -def llimpl_arena_munmap(arena_addr, nbytes): - from rpython.rlib import rmmap - assert nbytes >= 0 - rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) -register_external(arena_munmap, [llmemory.Address, int], None, - 'll_arena.arena_munmap', - llimpl=llimpl_arena_munmap, - llfakeimpl=arena_munmap, - sandboxsafe=True) + def llimpl_arena_munmap(arena_addr, nbytes): + from rpython.rlib import rmmap + assert nbytes >= 0 + rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) + register_external(arena_munmap, [llmemory.Address, int], None, + 'll_arena.arena_munmap', + llimpl=llimpl_arena_munmap, + llfakeimpl=arena_munmap, + sandboxsafe=True) def llimpl_arena_reset(arena_addr, size, zero): if zero: From pypy.commits at gmail.com Wed Feb 7 15:38:00 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Feb 2018 12:38:00 -0800 (PST) Subject: [pypy-commit] pypy default: disable arena_mmap, fix win32 translation? Message-ID: <5a7b63a8.0fe71c0a.48b41.e6a0@mx.google.com> Author: Matti Picus Branch: Changeset: r93783:4cf1cfb34efc Date: 2018-02-07 15:36 -0500 http://bitbucket.org/pypy/pypy/changeset/4cf1cfb34efc/ Log: disable arena_mmap, fix win32 translation? diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -9,6 +9,7 @@ WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT] assert 1 << WORD_POWER_2 == WORD +USE_MMAP = False # Terminology: the memory is subdivided into "arenas" containing "pages". # A page contains a number of allocated objects, called "blocks". @@ -296,7 +297,10 @@ # # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address - arena_base = llarena.arena_mmap(self.arena_size) + if USE_MMAP: + arena_base = llarena.arena_mmap(self.arena_size) + else: + arena_base = llarena.arena_malloc(self.arena_size, False) self.total_memory_alloced += self.arena_size self.peak_memory_alloced = max(self.total_memory_alloced, self.peak_memory_alloced) @@ -406,7 +410,11 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. - llarena.arena_munmap(arena.base, self.arena_size) + if USE_MMAP: + llarena.arena_munmap(arena.base, self.arena_size) + else: + llarena.arena_reset(arena.base, self.arena_size, 4) + llarena.arena_free(arena.base) self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import llmemory from rpython.rlib.rarithmetic import is_valid_int from rpython.rtyper.lltypesystem.lloperation import llop - +import os, sys # An "arena" is a large area of memory which can hold a number of # objects, not necessarily all of the same type or size. It's used by @@ -327,15 +327,16 @@ assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() -def arena_mmap(nbytes): - """Allocate and return a new arena, zero-initialized by the - system, calling mmap().""" - return arena_malloc(nbytes, True) +if os.name == 'posix': + def arena_mmap(nbytes): + """Allocate and return a new arena, zero-initialized by the + system, calling mmap().""" + return arena_malloc(nbytes, True) -def arena_munmap(arena_addr, nbytes): - """Release an arena allocated with arena_mmap().""" - arena_free(arena_addr) - assert nbytes == arena_addr.arena.nbytes + def arena_munmap(arena_addr, nbytes): + """Release an arena allocated with arena_mmap().""" + arena_free(arena_addr) + assert nbytes == arena_addr.arena.nbytes def arena_reset(arena_addr, size, zero): @@ -404,7 +405,6 @@ # We can tweak these implementations to be more suited to very large # chunks of memory. -import os, sys from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.extfunc import register_external from rpython.rtyper.tool.rffi_platform import memory_alignment @@ -541,30 +541,31 @@ llfakeimpl=arena_free, sandboxsafe=True) -def llimpl_arena_mmap(nbytes): - from rpython.rlib import rmmap - flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS - prot = rmmap.PROT_READ | rmmap.PROT_WRITE - p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( - lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) - if p == rffi.cast(llmemory.Address, -1): - p = rffi.cast(llmemory.Address, 0) - return p -register_external(arena_mmap, [int], llmemory.Address, - 'll_arena.arena_mmap', - llimpl=llimpl_arena_mmap, - llfakeimpl=arena_mmap, - sandboxsafe=True) +if os.name == 'posix': + def llimpl_arena_mmap(nbytes): + from rpython.rlib import rmmap + flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS + prot = rmmap.PROT_READ | rmmap.PROT_WRITE + p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( + lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) + if p == rffi.cast(llmemory.Address, -1): + p = rffi.cast(llmemory.Address, 0) + return p + register_external(arena_mmap, [int], llmemory.Address, + 'll_arena.arena_mmap', + llimpl=llimpl_arena_mmap, + llfakeimpl=arena_mmap, + sandboxsafe=True) -def llimpl_arena_munmap(arena_addr, nbytes): - from rpython.rlib import rmmap - assert nbytes >= 0 - rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) -register_external(arena_munmap, [llmemory.Address, int], None, - 'll_arena.arena_munmap', - llimpl=llimpl_arena_munmap, - llfakeimpl=arena_munmap, - sandboxsafe=True) + def llimpl_arena_munmap(arena_addr, nbytes): + from rpython.rlib import rmmap + assert nbytes >= 0 + rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) + register_external(arena_munmap, [llmemory.Address, int], None, + 'll_arena.arena_munmap', + llimpl=llimpl_arena_munmap, + llfakeimpl=arena_munmap, + sandboxsafe=True) def llimpl_arena_reset(arena_addr, size, zero): if zero: From pypy.commits at gmail.com Thu Feb 8 08:22:03 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 08 Feb 2018 05:22:03 -0800 (PST) Subject: [pypy-commit] pypy default: completely unmerge mmap-for-arenas, it's not ready Message-ID: <5a7c4efb.01c5df0a.e57c.1ba8@mx.google.com> Author: fijal Branch: Changeset: r93785:50125437a55e Date: 2018-02-08 14:21 +0100 http://bitbucket.org/pypy/pypy/changeset/50125437a55e/ Log: completely unmerge mmap-for-arenas, it's not ready diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -9,8 +9,6 @@ WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT] assert 1 << WORD_POWER_2 == WORD -USE_MMAP = False - # Terminology: the memory is subdivided into "arenas" containing "pages". # A page contains a number of allocated objects, called "blocks". @@ -297,10 +295,7 @@ # # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address - if USE_MMAP: - arena_base = llarena.arena_mmap(self.arena_size) - else: - arena_base = llarena.arena_malloc(self.arena_size, False) + arena_base = llarena.arena_malloc(self.arena_size, False) self.total_memory_alloced += self.arena_size self.peak_memory_alloced = max(self.total_memory_alloced, self.peak_memory_alloced) @@ -410,11 +405,8 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. - if USE_MMAP: - llarena.arena_munmap(arena.base, self.arena_size) - else: - llarena.arena_reset(arena.base, self.arena_size, 4) - llarena.arena_free(arena.base) + llarena.arena_reset(arena.base, self.arena_size, 4) + llarena.arena_free(arena.base) self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -327,18 +327,6 @@ assert not arena_addr.arena.objectptrs arena_addr.arena.mark_freed() -if os.name == 'posix': - def arena_mmap(nbytes): - """Allocate and return a new arena, zero-initialized by the - system, calling mmap().""" - return arena_malloc(nbytes, True) - - def arena_munmap(arena_addr, nbytes): - """Release an arena allocated with arena_mmap().""" - arena_free(arena_addr) - assert nbytes == arena_addr.arena.nbytes - - def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. This can also be used on a subrange of the arena. @@ -541,32 +529,6 @@ llfakeimpl=arena_free, sandboxsafe=True) -if os.name == 'posix': - def llimpl_arena_mmap(nbytes): - from rpython.rlib import rmmap - flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS - prot = rmmap.PROT_READ | rmmap.PROT_WRITE - p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe( - lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0)) - if p == rffi.cast(llmemory.Address, -1): - p = rffi.cast(llmemory.Address, 0) - return p - register_external(arena_mmap, [int], llmemory.Address, - 'll_arena.arena_mmap', - llimpl=llimpl_arena_mmap, - llfakeimpl=arena_mmap, - sandboxsafe=True) - - def llimpl_arena_munmap(arena_addr, nbytes): - from rpython.rlib import rmmap - assert nbytes >= 0 - rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes) - register_external(arena_munmap, [llmemory.Address, int], None, - 'll_arena.arena_munmap', - llimpl=llimpl_arena_munmap, - llfakeimpl=arena_munmap, - sandboxsafe=True) - def llimpl_arena_reset(arena_addr, size, zero): if zero: if zero == 1: From pypy.commits at gmail.com Thu Feb 8 08:27:39 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 08 Feb 2018 05:27:39 -0800 (PST) Subject: [pypy-commit] pypy default: maybe a fix for gc tests, we will see Message-ID: <5a7c504b.07c7df0a.bb39a.919b@mx.google.com> Author: fijal Branch: Changeset: r93786:cc02e211a1a3 Date: 2018-02-08 14:26 +0100 http://bitbucket.org/pypy/pypy/changeset/cc02e211a1a3/ Log: maybe a fix for gc tests, we will see diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -1,4 +1,4 @@ -from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup, rffi from rpython.rtyper import rclass from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.debug import ll_assert @@ -96,6 +96,8 @@ return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY)) def q_destructor_or_custom_trace(self, typeid): + if not self.get(typeid).customdata: + return lltype.nullptr(rffi.VOIDP.TO) return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): From pypy.commits at gmail.com Thu Feb 8 23:57:16 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 08 Feb 2018 20:57:16 -0800 (PST) Subject: [pypy-commit] pypy msvc14: tweak checks of file descriptors for lack of is_valid_fd, update tests Message-ID: <5a7d2a2c.6499df0a.75d79.ecd9@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93787:024b40ba48bf Date: 2018-02-08 23:56 -0500 http://bitbucket.org/pypy/pypy/changeset/024b40ba48bf/ Log: tweak checks of file descriptors for lack of is_valid_fd, update tests diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -115,13 +115,6 @@ with FdValidator(c_fileno(fp)): return _feof(fp) -def is_valid_fp(fp): - try: - with FdValidator(c_fileno(fp)): - return 1 - except IOError: - return 0 - pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -99,9 +99,7 @@ """fdopen(fd [, mode='r' [, buffering]]) -> file_object Return an open file object connected to a file descriptor.""" - from rpython.rlib.rposix import FdValidator - with FdValidator(fd): - return _fdopen(fd, mode, buffering) + return _fdopen(fd, mode, buffering) def tmpfile(): """Create a temporary file. diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -18,7 +18,7 @@ USEMODULES += ['fcntl'] else: # On windows, os.popen uses the subprocess module - USEMODULES += ['_rawffi', 'thread', 'signal'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) @@ -300,8 +300,13 @@ # There used to be code here to ensure that fcntl is not faked # but we can't do that cleanly any more - exc = raises(OSError, posix.fdopen, fd) - assert exc.value.errno == errno.EBADF + try: + fid = posix.fdopen(fd) + fid.read(10) + except IOError as e: + assert e.errno == errno.EBADF + else: + assert False, "using result of fdopen(fd) on closed file must raise" def test_fdopen_hackedbuiltins(self): "Same test, with __builtins__.file removed" @@ -331,8 +336,17 @@ path = self.path posix = self.posix fd = posix.open(path, posix.O_RDONLY) - exc = raises(OSError, posix.fdopen, fd, 'w') - assert str(exc.value) == "[Errno 22] Invalid argument" + # compatability issue - using Visual Studio 10 and above no + # longer raises on fid creation, only when _using_ fid + # win32 python2 raises IOError on flush(), win32 python3 raises OSError + try: + fid = posix.fdopen(fd, 'w') + fid.write('abc') + fid.flush() + except (OSError, IOError) as e: + assert e.errno in (9, 22) + else: + assert False, "expected OSError" posix.close(fd) # fd should not be closed def test_getcwd(self): From pypy.commits at gmail.com Fri Feb 9 02:57:31 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 08 Feb 2018 23:57:31 -0800 (PST) Subject: [pypy-commit] pypy default: fix Message-ID: <5a7d546b.49a0df0a.42f1c.8758@mx.google.com> Author: fijal Branch: Changeset: r93788:30ec49b828a4 Date: 2018-02-09 08:56 +0100 http://bitbucket.org/pypy/pypy/changeset/30ec49b828a4/ Log: fix diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -97,7 +97,7 @@ def q_destructor_or_custom_trace(self, typeid): if not self.get(typeid).customdata: - return lltype.nullptr(rffi.VOIDP.TO) + return lltype.nullptr(GCData.CUSTOM_DATA_STRUCT) return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): From pypy.commits at gmail.com Fri Feb 9 05:07:06 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 09 Feb 2018 02:07:06 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 Message-ID: <5a7d72ca.02b7df0a.a7e14.983d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.6 Changeset: r93789:9c18391110eb Date: 2018-02-09 11:06 +0100 http://bitbucket.org/pypy/pypy/changeset/9c18391110eb/ Log: merge py3.5 diff too long, truncating to 2000 out of 17345 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -44,3 +44,10 @@ d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,2 +1,3 @@ pytest hypothesis +vmprof diff --git a/extra_tests/test_import.py b/extra_tests/test_import.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_import.py @@ -0,0 +1,41 @@ +import pytest +import sys +import time +from _thread import start_new_thread + + at pytest.mark.xfail('__pypy__' not in sys.builtin_module_names, + reason='Fails on CPython') +def test_multithreaded_import(tmpdir): + tmpfile = tmpdir.join('multithreaded_import_test.py') + tmpfile.write('''if 1: + x = 666 + import time + for i in range(1000): time.sleep(0.001) + x = 42 + ''') + + oldpath = sys.path[:] + try: + sys.path.insert(0, str(tmpdir)) + got = [] + + def check(): + import multithreaded_import_test + got.append(getattr(multithreaded_import_test, 'x', '?')) + + for i in range(5): + start_new_thread(check, ()) + + for n in range(100): + for i in range(105): + time.sleep(0.001) + if len(got) == 5: + break + else: + raise AssertionError("got %r so far but still waiting" % + (got,)) + + assert got == [42] * 5 + + finally: + sys.path[:] = oldpath diff --git a/extra_tests/test_json.py b/extra_tests/test_json.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_json.py @@ -0,0 +1,29 @@ +import pytest +import json +from hypothesis import given, strategies + +def is_(x, y): + return type(x) is type(y) and x == y + +def test_no_ensure_ascii(): + assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') + assert is_(json.dumps(u"\xc0", ensure_ascii=False), u'"\xc0"') + with pytest.raises(TypeError): + json.dumps((u"\u1234", b"x"), ensure_ascii=False) + with pytest.raises(TypeError): + json.dumps((b"x", u"\u1234"), ensure_ascii=False) + +def test_issue2191(): + assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/3/json/encoder.py b/lib-python/3/json/encoder.py --- a/lib-python/3/json/encoder.py +++ b/lib-python/3/json/encoder.py @@ -2,18 +2,8 @@ """ import re -try: - from _json import encode_basestring_ascii as c_encode_basestring_ascii -except ImportError: - c_encode_basestring_ascii = None -try: - from _json import encode_basestring as c_encode_basestring -except ImportError: - c_encode_basestring = None -try: - from _json import make_encoder as c_make_encoder -except ImportError: - c_make_encoder = None +from __pypy__.builders import StringBuilder + ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') @@ -33,19 +23,17 @@ INFINITY = float('inf') -def py_encode_basestring(s): +def raw_encode_basestring(s): """Return a JSON representation of a Python string """ def replace(match): return ESCAPE_DCT[match.group(0)] - return '"' + ESCAPE.sub(replace, s) + '"' + return ESCAPE.sub(replace, s) +encode_basestring = lambda s: '"' + raw_encode_basestring(s) + '"' -encode_basestring = (c_encode_basestring or py_encode_basestring) - - -def py_encode_basestring_ascii(s): +def raw_encode_basestring_ascii(s): """Return an ASCII-only JSON representation of a Python string """ @@ -64,12 +52,10 @@ s1 = 0xd800 | ((n >> 10) & 0x3ff) s2 = 0xdc00 | (n & 0x3ff) return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '"' + ESCAPE_ASCII.sub(replace, s) + '"' + return ESCAPE_ASCII.sub(replace, s) +encode_basestring_ascii = lambda s: '"' + raw_encode_basestring_ascii(s) + '"' -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - class JSONEncoder(object): """Extensible JSON encoder for Python data structures. @@ -146,6 +132,10 @@ self.skipkeys = skipkeys self.ensure_ascii = ensure_ascii + if ensure_ascii: + self.__encoder = raw_encode_basestring_ascii + else: + self.__encoder = raw_encode_basestring self.check_circular = check_circular self.allow_nan = allow_nan self.sort_keys = sort_keys @@ -157,6 +147,11 @@ if default is not None: self.default = default + if indent is not None and not isinstance(indent, str): + self.indent_str = ' ' * indent + else: + self.indent_str = indent + def default(self, o): """Implement this method in a subclass such that it returns a serializable object for ``o``, or calls the base implementation @@ -187,19 +182,126 @@ '{"foo": ["bar", "baz"]}' """ - # This is for extremely simple cases and benchmarks. + if self.check_circular: + markers = {} + else: + markers = None + builder = StringBuilder() + self.__encode(o, markers, builder, 0) + return builder.build() + + def __emit_indent(self, builder, _current_indent_level): + if self.indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + self.indent_str * _current_indent_level + separator = self.item_separator + newline_indent + builder.append(newline_indent) + else: + separator = self.item_separator + return separator, _current_indent_level + + def __emit_unindent(self, builder, _current_indent_level): + if self.indent is not None: + builder.append('\n') + builder.append(self.indent_str * (_current_indent_level - 1)) + + def __encode(self, o, markers, builder, _current_indent_level): if isinstance(o, str): - if self.ensure_ascii: - return encode_basestring_ascii(o) + builder.append('"') + builder.append(self.__encoder(o)) + builder.append('"') + elif o is None: + builder.append('null') + elif o is True: + builder.append('true') + elif o is False: + builder.append('false') + elif isinstance(o, int): + # Subclasses of int/float may override __str__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + builder.append(int.__str__(o)) + elif isinstance(o, float): + builder.append(self.__floatstr(o)) + elif isinstance(o, (list, tuple)): + if not o: + builder.append('[]') + return + self.__encode_list(o, markers, builder, _current_indent_level) + elif isinstance(o, dict): + if not o: + builder.append('{}') + return + self.__encode_dict(o, markers, builder, _current_indent_level) + else: + self.__mark_markers(markers, o) + res = self.default(o) + self.__encode(res, markers, builder, _current_indent_level) + self.__remove_markers(markers, o) + return res + + def __encode_list(self, l, markers, builder, _current_indent_level): + self.__mark_markers(markers, l) + builder.append('[') + first = True + separator, _current_indent_level = self.__emit_indent(builder, + _current_indent_level) + for elem in l: + if first: + first = False else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - return ''.join(chunks) + builder.append(separator) + self.__encode(elem, markers, builder, _current_indent_level) + del elem # XXX grumble + self.__emit_unindent(builder, _current_indent_level) + builder.append(']') + self.__remove_markers(markers, l) + + def __encode_dict(self, d, markers, builder, _current_indent_level): + self.__mark_markers(markers, d) + first = True + builder.append('{') + separator, _current_indent_level = self.__emit_indent(builder, + _current_indent_level) + if self.sort_keys: + items = sorted(d.items(), key=lambda kv: kv[0]) + else: + items = d.items() + + for key, v in items: + if first: + first = False + else: + builder.append(separator) + if isinstance(key, str): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + key = self.__floatstr(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif isinstance(key, int): + # see comment for int in __encode + key = int.__str__(key) + elif self.skipkeys: + continue + else: + raise TypeError("key " + repr(key) + " is not a string") + builder.append('"') + builder.append(self.__encoder(key)) + builder.append('"') + builder.append(self.key_separator) + self.__encode(v, markers, builder, _current_indent_level) + del key + del v # XXX grumble + self.__emit_unindent(builder, _current_indent_level) + builder.append('}') + self.__remove_markers(markers, d) def iterencode(self, o, _one_shot=False): """Encode the given object and yield each string @@ -215,83 +317,53 @@ markers = {} else: markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii + return self.__iterencode(o, markers, 0) + + def __floatstr(self, o): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + + if o != o: + text = 'NaN' + elif o == INFINITY: + text = 'Infinity' + elif o == -INFINITY: + text = '-Infinity' else: - _encoder = encode_basestring + return float.__repr__(o) - def floatstr(o, allow_nan=self.allow_nan, - _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on the - # internals. + if not self.allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) + return text - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) + def __mark_markers(self, markers, o): + if markers is not None: + if id(o) in markers: + raise ValueError("Circular reference detected") + markers[id(o)] = None - return text + def __remove_markers(self, markers, o): + if markers is not None: + del markers[id(o)] - - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot) - return _iterencode(o, 0) - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - ValueError=ValueError, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - _intstr=int.__str__, - ): - - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - - def _iterencode_list(lst, _current_indent_level): + def __iterencode_list(self, lst, markers, _current_indent_level): if not lst: yield '[]' return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst + self.__mark_markers(markers, lst) buf = '[' - if _indent is not None: + if self.indent is not None: _current_indent_level += 1 - newline_indent = '\n' + _indent * _current_indent_level - separator = _item_separator + newline_indent + newline_indent = '\n' + self.indent_str * _current_indent_level + separator = self.item_separator + newline_indent buf += newline_indent else: newline_indent = None - separator = _item_separator + separator = self.item_separator first = True for value in lst: if first: @@ -299,7 +371,7 @@ else: buf = separator if isinstance(value, str): - yield buf + _encoder(value) + yield buf + '"' + self.__encoder(value) + '"' elif value is None: yield buf + 'null' elif value is True: @@ -307,49 +379,44 @@ elif value is False: yield buf + 'false' elif isinstance(value, int): - # Subclasses of int/float may override __str__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) + # see comment for int in __encode + yield buf + int.__str__(value) elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) + yield buf + self.__floatstr(value) else: yield buf if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) + chunks = self.__iterencode_list(value, markers, + _current_indent_level) elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + chunks = self.__iterencode_dict(value, markers, + _current_indent_level) else: - chunks = _iterencode(value, _current_indent_level) + chunks = self.__iterencode(value, markers, + _current_indent_level) yield from chunks if newline_indent is not None: _current_indent_level -= 1 - yield '\n' + _indent * _current_indent_level + yield '\n' + self.indent_str * _current_indent_level yield ']' - if markers is not None: - del markers[markerid] + self.__remove_markers(markers, lst) - def _iterencode_dict(dct, _current_indent_level): + def __iterencode_dict(self, dct, markers, _current_indent_level): if not dct: yield '{}' return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct + self.__mark_markers(markers, dct) yield '{' - if _indent is not None: + if self.indent is not None: _current_indent_level += 1 - newline_indent = '\n' + _indent * _current_indent_level - item_separator = _item_separator + newline_indent + newline_indent = '\n' + self.indent_str * _current_indent_level + item_separator = self.item_separator + newline_indent yield newline_indent else: newline_indent = None - item_separator = _item_separator + item_separator = self.item_separator first = True - if _sort_keys: + if self.sort_keys: items = sorted(dct.items(), key=lambda kv: kv[0]) else: items = dct.items() @@ -359,8 +426,7 @@ # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. elif isinstance(key, float): - # see comment for int/float in _make_iterencode - key = _floatstr(key) + key = self.__floatstr(key) elif key is True: key = 'true' elif key is False: @@ -368,9 +434,9 @@ elif key is None: key = 'null' elif isinstance(key, int): - # see comment for int/float in _make_iterencode - key = _intstr(key) - elif _skipkeys: + # see comment for int in __encode + key = int.__str__(key) + elif self.skipkeys: continue else: raise TypeError("key " + repr(key) + " is not a string") @@ -378,10 +444,10 @@ first = False else: yield item_separator - yield _encoder(key) - yield _key_separator + yield '"' + self.__encoder(key) + '"' + yield self.key_separator if isinstance(value, str): - yield _encoder(value) + yield '"' + self.__encoder(value) + '"' elif value is None: yield 'null' elif value is True: @@ -389,29 +455,29 @@ elif value is False: yield 'false' elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) + yield int.__str__(value) elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) + yield self.__floatstr(value) else: if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) + chunks = self.__iterencode_list(value, markers, + _current_indent_level) elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + chunks = self.__iterencode_dict(value, markers, + _current_indent_level) else: - chunks = _iterencode(value, _current_indent_level) + chunks = self.__iterencode(value, markers, + _current_indent_level) yield from chunks if newline_indent is not None: _current_indent_level -= 1 - yield '\n' + _indent * _current_indent_level + yield '\n' + self.indent_str * _current_indent_level yield '}' - if markers is not None: - del markers[markerid] + self.__remove_markers(markers, dct) - def _iterencode(o, _current_indent_level): + def __iterencode(self, o, markers, _current_indent_level): if isinstance(o, str): - yield _encoder(o) + yield '"' + self.__encoder(o) + '"' elif o is None: yield 'null' elif o is True: @@ -419,23 +485,22 @@ elif o is False: yield 'false' elif isinstance(o, int): - # see comment for int/float in _make_iterencode - yield _intstr(o) + yield int.__str__(o) elif isinstance(o, float): - # see comment for int/float in _make_iterencode - yield _floatstr(o) + yield self.__floatstr(o) elif isinstance(o, (list, tuple)): - yield from _iterencode_list(o, _current_indent_level) + yield from self.__iterencode_list(o, markers, _current_indent_level) elif isinstance(o, dict): - yield from _iterencode_dict(o, _current_indent_level) + yield from self.__iterencode_dict(o, markers, _current_indent_level) else: - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) - if markers is not None: - del markers[markerid] - return _iterencode + self.__mark_markers(markers, o) + obj = self.default(o) + yield from self.__iterencode(obj, markers, _current_indent_level) + self.__remove_markers(markers, o) + + +# overwrite some helpers here with more efficient versions +try: + from _pypyjson import raw_encode_basestring_ascii +except ImportError: + pass diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py --- a/lib-python/3/subprocess.py +++ b/lib-python/3/subprocess.py @@ -1614,7 +1614,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy3-c.so', 'libpypy3-c.dylib']: + for libname in ['libpypy3-c.so', 'libpypy3-c.dylib', 'libpypy3-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/3/test/test_compile.py b/lib-python/3/test/test_compile.py --- a/lib-python/3/test/test_compile.py +++ b/lib-python/3/test/test_compile.py @@ -527,7 +527,8 @@ with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + # PyPy change: we have a different error here + self.assertIn(b"SyntaxError", res.err) def test_yet_more_evil_still_undecodable(self): # Issue #25388 diff --git a/lib-python/3/test/test_venv.py b/lib-python/3/test/test_venv.py --- a/lib-python/3/test/test_venv.py +++ b/lib-python/3/test/test_venv.py @@ -45,6 +45,8 @@ os.readlink(executable)) except OSError: pass + except AttributeError: + pass return executable diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py --- a/lib-python/3/venv/__init__.py +++ b/lib-python/3/venv/__init__.py @@ -111,13 +111,17 @@ executable = sys.executable # # PyPy extension: resolve 'executable' if it is a symlink - try: - for i in range(10): - executable = os.path.abspath(executable) - executable = os.path.join(os.path.dirname(executable), - os.readlink(executable)) - except OSError: - pass + # XXX as of PyPy 5.10, win32 does not have readlink + # note it is highly unlikely that symlinks were used on win32 + # since it requires admin priveleges + if hasattr(os, 'readlink'): + try: + for i in range(10): + executable = os.path.abspath(executable) + executable = os.path.join(os.path.dirname(executable), + os.readlink(executable)) + except OSError: + pass # dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -62,7 +62,7 @@ self.basename = basename self._usemodules = usemodules.split() + [ '_socket', 'binascii', 'time', - 'select', 'signal', 'faulthandler'] + 'select', 'signal', ] if not sys.platform == 'win32': self._usemodules.extend(['_posixsubprocess', 'fcntl']) self._compiler = compiler diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -20,9 +20,15 @@ SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE, pyerr_write_unraisable) from _cffi_ssl._stdssl import error -from select import poll, POLLIN, POLLOUT, select +from select import select from enum import IntEnum as _IntEnum +if sys.platform == 'win32': + HAVE_POLL = False +else: + from select import poll, POLLIN, POLLOUT + HAVE_POLL = True + OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8') OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER ver = OPENSSL_VERSION_NUMBER @@ -158,8 +164,6 @@ def _monotonic_clock(): return time.clock_gettime(time.CLOCK_MONOTONIC) -HAVE_POLL = True - def _ssl_select(sock, writing, timeout): if HAVE_POLL: p = poll() diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -14,3 +14,14 @@ # RAND_egd is optional and might not be available on e.g. libressl if hasattr(_stdssl, 'RAND_egd'): RAND_egd = builtinify(RAND_egd) + +import sys +if sys.platform == "win32" and 'enum_certificates' not in globals(): + def enum_certificates(*args, **kwds): + import warnings + warnings.warn("ssl.enum_certificates() is not implemented") + return [] + def enum_crls(*args, **kwds): + import warnings + warnings.warn("ssl.enum_crls() is not implemented") + return [] diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -164,6 +164,7 @@ SW_HIDE = 0 INFINITE = 0xffffffff WAIT_OBJECT_0 = 0 +WAIT_ABANDONED_0 = 0x80 WAIT_TIMEOUT = 0x102 CREATE_NEW_CONSOLE = 0x010 CREATE_NEW_PROCESS_GROUP = 0x200 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/faulthandler.py b/lib_pypy/faulthandler.py new file mode 100644 --- /dev/null +++ b/lib_pypy/faulthandler.py @@ -0,0 +1,9 @@ +# This is only imported for platforms where the built-in faulthandler module is not +# available. It provides no function at all so far, but it is enough to start the +# CPython test suite. + +def enable(*args, **kwargs): + pass + +def register(*args, **kwargs): + pass diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -127,7 +127,7 @@ return None if self.__main: self = getcurrent() - f = _continulet.__reduce__(self)[2][0] + f = self._get_frame() if not f: return None return f.f_back.f_back.f_back # go past start(), __switch(), switch() diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py --- a/lib_pypy/pyrepl/reader.py +++ b/lib_pypy/pyrepl/reader.py @@ -50,11 +50,12 @@ def _my_unctrl(c, u=_make_unctrl_map()): + # takes an integer, returns a unicode if c in u: return u[c] else: if unicodedata.category(c).startswith('C'): - return br'\u%04x' % ord(c) + return r'\u%04x' % ord(c) else: return c diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -500,7 +500,7 @@ os.write(self.output_fd, fmt[:x]) fmt = fmt[y:] delay = int(m.group(1)) - if '*' in m.group(2): + if b'*' in m.group(2): delay *= self.height if self._pad: nchars = (bps*delay)/1000 diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -149,7 +149,7 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X:: +On Mac OS X: Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -362,7 +362,11 @@ containers (as list items or in sets for example), the exact rule of equality used is "``if x is y or x == y``" (on both CPython and PyPy); as a consequence, because all ``nans`` are identical in PyPy, you -cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__). +Another consequence is that ``cmp(float('nan'), float('nan')) == 0``, because +``cmp`` checks with ``is`` first whether the arguments are identical (there is +no good value to return from this call to ``cmp``, because ``cmp`` pretends +that there is a total order on floats, but that is wrong for NaNs). .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of @@ -554,6 +558,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,31 +1,16 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:d56dadcef996 - -.. branch: cppyy-packaging -Cleanup and improve cppyy packaging - -.. branch: docs-osx-brew-openssl - -.. branch: keep-debug-symbols -Add a smartstrip tool, which can optionally keep the debug symbols in a -separate file, instead of just stripping them away. Use it in packaging - -.. branch: bsd-patches -Fix failures on FreeBSD, contributed by David Naylor as patches on the issue -tracker (issues 2694, 2695, 2696, 2697) - -.. branch: run-extra-tests -Run extra_tests/ in buildbot - -.. branch: vmprof-0.4.10 -Upgrade the _vmprof backend to vmprof 0.4.10 - -.. branch: fix-vmprof-stacklet-switch -Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) - -.. branch: win32-vcvars - +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -0,0 +1,42 @@ +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst --- a/pypy/doc/whatsnew-pypy2-5.6.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst @@ -107,7 +107,7 @@ .. branch: newinitwarn -Match CPython's stricter handling of __new/init__ arguments +Match CPython's stricter handling of ``__new__``/``__init__`` arguments .. branch: openssl-1.1 diff --git a/pypy/doc/whatsnew-pypy3-5.10.0.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.10.0.rst @@ -0,0 +1,21 @@ +========================= +What's new in PyPy3 5.9+ +========================= + +.. this is the revision after release-pypy3.5-5.9 +.. startrev: be41e3ac0a29 + +.. branch: sched_yield +Add sched_yield posix attribute + +.. branch: py3.5-appexec +Raise if space.is_true(space.appexec()) used in app level tests, fix tests +that did this + +.. branch: py3.5-mac-embedding +Download and patch dependencies when building cffi-based stdlib modules + +.. branch: os_lockf + +.. branch: py3.5-xattr +Add posix.*attr() functions diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,18 +1,16 @@ ========================= -What's new in PyPy3 5.9+ +What's new in PyPy3 5.10+ ========================= -.. this is the revision after release-pypy3.5-5.9 -.. startrev: be41e3ac0a29 +.. this is the revision after release-pypy3.5-v5.10 +.. startrev: 34c63fba0bba -.. branch: sched_yield -Add sched_yield posix attribute +.. branch: hroncok/fix-typeerror-str-does-not-support-the-b-1514414905375 -.. branch: py3.5-appexec -Raise if space.is_true(space.appexec()) used in app level tests, fix tests -that did this +Fix for bytestrings in console repl -.. branch: py3.5-mac-embedding -Download and patch dependencies when building cffi-based stdlib modules +.. branch: py3-winreg -.. branch: os_lockf +Update winreg module to use unicode, wide-strings + + diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -11,7 +11,7 @@ To build pypy-c you need a working python environment, and a C compiler. It is possible to translate with a CPython 2.6 or later, but this is not -the preferred way, because it will take a lot longer to run � depending +the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -103,6 +103,7 @@ must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the ``...\9.0\VC`` directory, and edit it, changing the lines that set ``VCINSTALLDIR`` and ``WindowsSdkDir``:: + set VCINSTALLDIR=%~dp0\ set WindowsSdkDir=%~dp0\..\WinSDK\ diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -574,10 +574,6 @@ except ValueError: pass # ignore "2 is not a valid file descriptor" - if we_are_translated(): - import __pypy__ - __pypy__.save_module_content_for_future_reload(sys) - mainmodule = type(sys)('__main__') mainmodule.__loader__ = sys.__loader__ mainmodule.__builtins__ = os.__builtins__ diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import specialize, we_are_translated from pypy.interpreter.astcompiler import ast, consts, misc, symtable @@ -269,7 +269,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -279,9 +279,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py --- a/pypy/interpreter/astcompiler/fstring.py +++ b/pypy/interpreter/astcompiler/fstring.py @@ -342,10 +342,11 @@ encoding = astbuilder.compile_info.encoding joined_pieces = [] fmode = False - try: - for i in range(atom_node.num_children()): + for i in range(atom_node.num_children()): + child = atom_node.get_child(i) + try: w_next = parsestring.parsestr( - space, encoding, atom_node.get_child(i).get_value()) + space, encoding, child.get_value()) if not isinstance(w_next, parsestring.W_FString): add_constant_string(astbuilder, joined_pieces, w_next, atom_node) @@ -353,17 +354,20 @@ parse_f_string(astbuilder, joined_pieces, w_next, atom_node) fmode = True - except error.OperationError as e: - if e.match(space, space.w_UnicodeError): - kind = 'unicode error' - elif e.match(space, space.w_ValueError): - kind = 'value error' - else: - raise - # Unicode/ValueError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise astbuilder.error('(%s) %s' % (kind, errmsg), atom_node) + except error.OperationError as e: + if e.match(space, space.w_UnicodeError): + kind = '(unicode error) ' + elif e.match(space, space.w_ValueError): + kind = '(value error) ' + elif e.match(space, space.w_SyntaxError): + kind = '' + else: + raise + # Unicode/ValueError/SyntaxError (without position information) in + # literal: turn into SyntaxError with position information + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + raise astbuilder.error('%s%s' % (kind, errmsg), child) if not fmode and len(joined_pieces) == 1: # <= the common path return joined_pieces[0] # ast.Str, Bytes or FormattedValue diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1418,3 +1418,7 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-1: truncated \\xXX escape") + input = "u'\\x1'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-2: truncated \\xXX escape") diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -627,12 +627,12 @@ if rwin32.WIN32 and isinstance(e, WindowsError): winerror = e.winerror try: - msg = rwin32.FormatError(winerror) + msg = rwin32.FormatErrorW(winerror) except ValueError: - msg = 'Windows Error %d' % winerror + msg = u'Windows Error %d' % winerror w_errno = space.w_None w_winerror = space.newint(winerror) - w_msg = space.newtext(msg) + w_msg = space.newunicode(msg) else: errno = e.errno if errno == EINTR: diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -171,6 +171,9 @@ def visit_text0(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_unicode(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_fsencode(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -320,6 +323,9 @@ def visit_text0(self, typ): self.run_args.append("space.text0_w(%s)" % (self.scopenext(),)) + def visit_unicode(self, typ): + self.run_args.append("space.unicode_w(%s)" % (self.scopenext(),)) + def visit_fsencode(self, typ): self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),)) @@ -485,6 +491,9 @@ def visit_text(self, typ): self.unwrap.append("space.text_w(%s)" % (self.nextarg(),)) + def visit_unicode(self, typ): + self.unwrap.append("space.unicode_w(%s)" % (self.nextarg(),)) + def visit_text0(self, typ): self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),)) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import imp, struct, types, new, sys, os +import imp, struct, types, sys, os from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -80,7 +80,7 @@ class PyCode(eval.Code): "CPython-style code objects." _immutable_fields_ = ["_signature", "co_argcount", "co_kwonlyargcount", "co_cellvars[*]", - "co_code", "co_consts_w[*]", "co_filename", + "co_code", "co_consts_w[*]", "co_filename", "w_filename", "co_firstlineno", "co_flags", "co_freevars[*]", "co_lnotab", "co_names_w[*]", "co_nlocals", "co_stacksize", "co_varnames[*]", @@ -111,6 +111,7 @@ assert isinstance(filename, str) rstring.check_str0(filename) self.co_filename = filename + self.w_filename = space.newfilename(filename) self.co_name = name self.co_firstlineno = firstlineno self.co_lnotab = lnotab @@ -203,6 +204,7 @@ if lastdirname: basename = '%s/%s' % (lastdirname, basename) self.co_filename = '/%s' % (basename,) + self.w_filename = self.space.newfilename(self.co_filename) co_names = property(lambda self: [self.space.str_w(w_name) for w_name in self.co_names_w]) # for trace @@ -427,7 +429,7 @@ space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([space.newtext(v) for v in self.co_varnames]), - space.newtext(self.co_filename), + self.w_filename, space.newtext(self.co_name), space.newint(self.co_firstlineno), space.newbytes(self.co_lnotab), @@ -451,7 +453,7 @@ space = self.space # co_name should be an identifier name = self.co_name.decode('utf-8') - fn = space.fsdecode_w(space.newbytes(self.co_filename)) + fn = space.unicode_w(self.w_filename) return space.newunicode(u'' % ( name, unicode(self.getaddrstring(space)), fn, -1 if self.co_firstlineno == 0 else self.co_firstlineno)) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -1007,8 +1007,9 @@ compile(b"# coding: latin1\n#\xfd\n", "dummy", "exec") raises(SyntaxError, compile, b"# coding: utf-8\n'\xfd'\n", "dummy", "exec") #1 - raises(SyntaxError, compile, b'# coding: utf-8\nx=5\nb"\xfd"\n', + excinfo = raises(SyntaxError, compile, b'# coding: utf-8\nx=5\nb"\xfd"\n', "dummy", "exec") #2 + assert excinfo.value.lineno == 3 # the following example still fails on CPython 3.5.2, skip if -A if '__pypy__' in sys.builtin_module_names: raises(SyntaxError, compile, b"# coding: utf-8\n#\xfd\n", @@ -1023,6 +1024,12 @@ raises(SyntaxError, compile, b"#\x00\nx=5#\xfd\n", "dummy", "exec", PyCF_ACCEPT_NULL_BYTES) + def test_correct_offset_in_many_bytes(self): + excinfo = raises(SyntaxError, compile, b'# coding: utf-8\nx = b"a" b"c" b"\xfd"\n', + "dummy", "exec") + assert excinfo.value.lineno == 2 + assert excinfo.value.offset == 14 + def test_dict_and_set_literal_order(self): x = 1 l1 = list({1:'a', 3:'b', 2:'c', 4:'d'}) diff --git a/pypy/interpreter/test/test_timeutils.py b/pypy/interpreter/test/test_timeutils.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/test_timeutils.py @@ -0,0 +1,13 @@ +import pytest +from rpython.rlib.rarithmetic import r_longlong +from pypy.interpreter.error import OperationError +from pypy.interpreter.timeutils import timestamp_w + +def test_timestamp_w(space): + w_1_year = space.newint(365 * 24 * 3600) + result = timestamp_w(space, w_1_year) + assert isinstance(result, r_longlong) + assert result // 10 ** 9 == space.int_w(w_1_year) + w_millenium = space.mul(w_1_year, space.newint(1000)) + with pytest.raises(OperationError): # timestamps overflow after ~300 years + timestamp_w(space, w_millenium) diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,5 +1,8 @@ import py -from pypy.interpreter.unicodehelper import encode_utf8, decode_utf8 +import pytest +import struct +from pypy.interpreter.unicodehelper import ( + encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be) from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp @@ -67,3 +70,26 @@ assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8sp(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] + + at pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) +def test_utf32_surrogates(unich): + assert (unicode_encode_utf_32_be(unich, 1, None) == + struct.pack('>i', ord(unich))) + with pytest.raises(UnicodeEncodeError): + unicode_encode_utf_32_be(unich, 1, None, allow_surrogates=False) + + def replace_with(ru, rs): + def errorhandler(errors, enc, msg, u, startingpos, endingpos): + if errors == 'strict': + raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg) + return ru, rs, endingpos + return unicode_encode_utf_32_be( + u"<%s>" % unich, 3, None, + errorhandler, allow_surrogates=False) + + assert replace_with(u'rep', None) == u''.encode('utf-32-be') + assert (replace_with(None, '\xca\xfe\xca\xfe') == + '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>') + + with pytest.raises(UnicodeDecodeError): + str_decode_utf_32_be(b"\x00\x00\xdc\x80", 4, None) diff --git a/pypy/interpreter/timeutils.py b/pypy/interpreter/timeutils.py --- a/pypy/interpreter/timeutils.py +++ b/pypy/interpreter/timeutils.py @@ -1,6 +1,15 @@ """ Access to the time module's high-resolution monotonic clock """ +import math +from rpython.rlib.rarithmetic import ( + r_longlong, ovfcheck_float_to_longlong) +from rpython.rlib import rfloat +from pypy.interpreter.error import oefmt + +SECS_TO_NS = 10 ** 9 +MS_TO_NS = 10 ** 6 +US_TO_NS = 10 ** 3 def monotonic(space): from pypy.module.time import interp_time @@ -9,3 +18,23 @@ else: w_res = interp_time.gettimeofday(space) return space.float_w(w_res) # xxx back and forth + +def timestamp_w(space, w_secs): + if space.isinstance_w(w_secs, space.w_float): + secs = space.float_w(w_secs) + if math.isnan(secs): + raise oefmt(space.w_ValueError, "timestamp is nan") + result_float = math.ceil(secs * SECS_TO_NS) + try: + return ovfcheck_float_to_longlong(result_float) + except OverflowError: + raise oefmt(space.w_OverflowError, + "timestamp %R too large to convert to C _PyTime_t", w_secs) + else: + try: + sec = space.bigint_w(w_secs).tolonglong() + result = sec * r_longlong(SECS_TO_NS) + except OverflowError: + raise oefmt(space.w_OverflowError, + "timestamp %R too large to convert to C _PyTime_t", w_secs) + return result diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -627,7 +627,7 @@ co_varnames = GetSetProperty(PyCode.fget_co_varnames), co_freevars = GetSetProperty(PyCode.fget_co_freevars), co_cellvars = GetSetProperty(PyCode.fget_co_cellvars), - co_filename = interp_attrproperty('co_filename', cls=PyCode, wrapfn="newfilename"), + co_filename = interp_attrproperty_w('w_filename', cls=PyCode), co_name = interp_attrproperty('co_name', cls=PyCode, wrapfn="newtext"), co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode, wrapfn="newint"), co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode, wrapfn="newbytes"), diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,8 +1,13 @@ import sys from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib import runicode -from pypy.module._codecs import interp_codecs +from rpython.rlib.runicode import ( + default_unicode_error_encode, default_unicode_error_decode, + MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR) + _WIN32 = sys.platform == 'win32' _MACOSX = sys.platform == 'darwin' if _WIN32: @@ -40,6 +45,7 @@ # ____________________________________________________________ def fsdecode(space, w_string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) if _WIN32: bytes = space.bytes_w(w_string) @@ -70,6 +76,7 @@ return space.newunicode(uni) def fsencode(space, w_uni): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) if _WIN32: uni = space.unicode_w(w_uni) @@ -107,6 +114,7 @@ # These functions take and return unwrapped rpython strings and unicodes def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result, consumed = runicode.str_decode_unicode_escape( @@ -157,3 +165,207 @@ # encoding error, it should always be reversible, and the reverse is # encode_utf8sp(). return decode_utf8(space, string, allow_surrogates=True) + +# ____________________________________________________________ +# utf-32 + +def str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2, + allow_surrogates=False) + return result, length + +def str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be', + allow_surrogates=False) + return result, length + +def str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le', + allow_surrogates=False) + return result, length + +BOM32_DIRECT = intmask(0x0000FEFF) +BOM32_REVERSE = intmask(0xFFFE0000) + +def str_decode_utf_32_helper(s, size, errors, final=True, + errorhandler=None, + byteorder="native", + public_encoding_name='utf32', + allow_surrogates=True): + if errorhandler is None: + errorhandler = default_unicode_error_decode + bo = 0 + From pypy.commits at gmail.com Fri Feb 9 09:00:07 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 06:00:07 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5a7da967.01da1c0a.2ae4f.130e@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93790:4e598c42abcc Date: 2018-02-09 08:59 -0500 http://bitbucket.org/pypy/pypy/changeset/4e598c42abcc/ Log: merge default into py3.5 diff too long, truncating to 2000 out of 4107 lines diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -155,9 +155,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements, uri) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements, uri) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.3 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,37 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -2,7 +2,7 @@ import __pypy__ import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,29 @@ .. branch: cpyext-datetime2 Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1,23 +1,15 @@ from pypy.interpreter.astcompiler import ast, consts, misc from pypy.interpreter.astcompiler import asthelpers # Side effects -from pypy.interpreter.astcompiler import fstring from pypy.interpreter import error from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError -from rpython.rlib.objectmodel import always_inline, we_are_translated +from pypy.interpreter.pyparser import parsestring +from rpython.rlib.objectmodel import specialize -def ast_from_node(space, node, compile_info, recursive_parser=None): +def ast_from_node(space, node, compile_info): """Turn a parse tree, node, to AST.""" - ast = ASTBuilder(space, node, compile_info, recursive_parser).build_ast() - # - # When we are not translated, we send this ast to validate_ast. - # The goal is to check that validate_ast doesn't crash on valid - # asts, at least. - if not we_are_translated(): - from pypy.interpreter.astcompiler import validate - validate.validate_ast(space, ast) - return ast + return ASTBuilder(space, node, compile_info).build_ast() augassign_operator_map = { @@ -26,7 +18,6 @@ '/=' : ast.Div, '//=' : ast.FloorDiv, '%=' : ast.Mod, - '@=' : ast.MatMult, '<<=' : ast.LShift, '>>=' : ast.RShift, '&=' : ast.BitAnd, @@ -47,18 +38,16 @@ tokens.STAR : ast.Mult, tokens.SLASH : ast.Div, tokens.DOUBLESLASH : ast.FloorDiv, - tokens.PERCENT : ast.Mod, - tokens.AT : ast.MatMult + tokens.PERCENT : ast.Mod }) class ASTBuilder(object): - def __init__(self, space, n, compile_info, recursive_parser=None): + def __init__(self, space, n, compile_info): self.space = space self.compile_info = compile_info self.root_node = n - self.recursive_parser = recursive_parser def build_ast(self): """Convert an top level parse tree node into an AST mod.""" @@ -129,9 +118,6 @@ except misc.ForbiddenNameAssignment as e: self.error("cannot assign to %s" % (e.name,), node) - def new_identifier(self, name): - return misc.new_identifier(self.space, name) - def set_context(self, expr, ctx): """Set the context of an expression to Store or Del if possible.""" try: @@ -141,6 +127,23 @@ except misc.ForbiddenNameAssignment as e: self.error_ast("cannot assign to %s" % (e.name,), e.node) + def handle_print_stmt(self, print_node): + dest = None + expressions = None + newline = True + start = 1 + child_count = print_node.num_children() + if child_count > 2 and print_node.get_child(1).type == tokens.RIGHTSHIFT: + dest = self.handle_expr(print_node.get_child(2)) + start = 4 + if (child_count + 1 - start) // 2: + expressions = [self.handle_expr(print_node.get_child(i)) + for i in range(start, child_count, 2)] + if print_node.get_child(-1).type == tokens.COMMA: + newline = False + return ast.Print(dest, expressions, newline, print_node.get_lineno(), + print_node.get_column()) + def handle_del_stmt(self, del_node): targets = self.handle_exprlist(del_node.get_child(1), ast.Del) return ast.Delete(targets, del_node.get_lineno(), del_node.get_column()) @@ -163,13 +166,17 @@ return ast.Return(values, flow_node.get_lineno(), flow_node.get_column()) elif first_child_type == syms.raise_stmt: exc = None - cause = None + value = None + traceback = None child_count = first_child.num_children() if child_count >= 2: exc = self.handle_expr(first_child.get_child(1)) if child_count >= 4: - cause = self.handle_expr(first_child.get_child(3)) - return ast.Raise(exc, cause, flow_node.get_lineno(), flow_node.get_column()) + value = self.handle_expr(first_child.get_child(3)) + if child_count == 6: + traceback = self.handle_expr(first_child.get_child(5)) + return ast.Raise(exc, value, traceback, flow_node.get_lineno(), + flow_node.get_column()) else: raise AssertionError("unknown flow statement") @@ -177,10 +184,9 @@ while True: import_name_type = import_name.type if import_name_type == syms.import_as_name: - name = self.new_identifier(import_name.get_child(0).get_value()) + name = import_name.get_child(0).get_value() if import_name.num_children() == 3: - as_name = self.new_identifier( - import_name.get_child(2).get_value()) + as_name = import_name.get_child(2).get_value() self.check_forbidden_name(as_name, import_name.get_child(2)) else: as_name = None @@ -193,12 +199,12 @@ alias = self.alias_for_import_name(import_name.get_child(0), store=False) asname_node = import_name.get_child(2) - alias.asname = self.new_identifier(asname_node.get_value()) + alias.asname = asname_node.get_value() self.check_forbidden_name(alias.asname, asname_node) return alias elif import_name_type == syms.dotted_name: if import_name.num_children() == 1: - name = self.new_identifier(import_name.get_child(0).get_value()) + name = import_name.get_child(0).get_value() if store: self.check_forbidden_name(name, import_name.get_child(0)) return ast.alias(name, None) @@ -226,15 +232,11 @@ dot_count = 0 while i < child_count: child = import_node.get_child(i) - child_type = child.type - if child_type == syms.dotted_name: + if child.type == syms.dotted_name: module = self.alias_for_import_name(child, False) i += 1 break - elif child_type == tokens.ELLIPSIS: - # Special case for tokenization. - dot_count += 2 - elif child_type != tokens.DOT: + elif child.type != tokens.DOT: break i += 1 dot_count += 1 @@ -266,14 +268,28 @@ raise AssertionError("unknown import node") def handle_global_stmt(self, global_node): - names = [self.new_identifier(global_node.get_child(i).get_value()) + names = [global_node.get_child(i).get_value() for i in range(1, global_node.num_children(), 2)] return ast.Global(names, global_node.get_lineno(), global_node.get_column()) - def handle_nonlocal_stmt(self, nonlocal_node): - names = [self.new_identifier(nonlocal_node.get_child(i).get_value()) - for i in range(1, nonlocal_node.num_children(), 2)] - return ast.Nonlocal(names, nonlocal_node.get_lineno(), nonlocal_node.get_column()) + def handle_exec_stmt(self, exec_node): + child_count = exec_node.num_children() + globs = None + locs = None + to_execute = self.handle_expr(exec_node.get_child(1)) + if child_count < 4: + if isinstance(to_execute, ast.Tuple) and \ + (len(to_execute.elts) == 2 or len(to_execute.elts) == 3): + globs = to_execute.elts[1] + if len(to_execute.elts) == 3: + locs = to_execute.elts[2] + to_execute = to_execute.elts[0] + elif child_count >= 4: + globs = self.handle_expr(exec_node.get_child(3)) + if child_count == 6: + locs = self.handle_expr(exec_node.get_child(5)) + return ast.Exec(to_execute, globs, locs, exec_node.get_lineno(), + exec_node.get_column()) def handle_assert_stmt(self, assert_node): expr = self.handle_expr(assert_node.get_child(1)) @@ -363,7 +379,7 @@ return ast.While(loop_test, body, otherwise, while_node.get_lineno(), while_node.get_column()) - def handle_for_stmt(self, for_node, is_async): + def handle_for_stmt(self, for_node): target_node = for_node.get_child(1) target_as_exprlist = self.handle_exprlist(target_node, ast.Store) if target_node.num_children() == 1: @@ -377,25 +393,21 @@ otherwise = self.handle_suite(for_node.get_child(8)) else: otherwise = None - if is_async: - return ast.AsyncFor(target, expr, body, otherwise, for_node.get_lineno(), - for_node.get_column()) - else: - return ast.For(target, expr, body, otherwise, for_node.get_lineno(), - for_node.get_column()) + return ast.For(target, expr, body, otherwise, for_node.get_lineno(), + for_node.get_column()) def handle_except_clause(self, exc, body): test = None - name = None + target = None suite = self.handle_suite(body) child_count = exc.num_children() if child_count >= 2: test = self.handle_expr(exc.get_child(1)) if child_count == 4: - name_node = exc.get_child(3) - name = self.new_identifier(name_node.get_value()) - self.check_forbidden_name(name, name_node) - return ast.ExceptHandler(test, name, suite, exc.get_lineno(), exc.get_column()) + target_child = exc.get_child(3) + target = self.handle_expr(target_child) + self.set_context(target, ast.Store) + return ast.ExceptHandler(test, target, suite, exc.get_lineno(), exc.get_column()) def handle_try_stmt(self, try_node): body = self.handle_suite(try_node.get_child(2)) @@ -415,100 +427,70 @@ else: otherwise = self.handle_suite(try_node.get_child(-1)) except_count -= 1 - handlers = [] if except_count: + handlers = [] for i in range(except_count): base_offset = i * 3 exc = try_node.get_child(3 + base_offset) except_body = try_node.get_child(5 + base_offset) handlers.append(self.handle_except_clause(exc, except_body)) - return ast.Try(body, handlers, otherwise, finally_suite, - try_node.get_lineno(), try_node.get_column()) + except_ast = ast.TryExcept(body, handlers, otherwise, + try_node.get_lineno(), try_node.get_column()) + if finally_suite is None: + return except_ast + body = [except_ast] + return ast.TryFinally(body, finally_suite, try_node.get_lineno(), + try_node.get_column()) - def handle_with_item(self, item_node): - test = self.handle_expr(item_node.get_child(0)) - if item_node.num_children() == 3: - target = self.handle_expr(item_node.get_child(2)) - self.set_context(target, ast.Store) - else: - target = None - return ast.withitem(test, target) - - def handle_with_stmt(self, with_node, is_async): + def handle_with_stmt(self, with_node): body = self.handle_suite(with_node.get_child(-1)) - items = [self.handle_with_item(with_node.get_child(i)) - for i in range(1, with_node.num_children()-2, 2)] - if is_async: - return ast.AsyncWith(items, body, with_node.get_lineno(), - with_node.get_column()) - else: - return ast.With(items, body, with_node.get_lineno(), - with_node.get_column()) + i = with_node.num_children() - 1 + while True: + i -= 2 + item = with_node.get_child(i) + test = self.handle_expr(item.get_child(0)) + if item.num_children() == 3: + target = self.handle_expr(item.get_child(2)) + self.set_context(target, ast.Store) + else: + target = None + wi = ast.With(test, target, body, with_node.get_lineno(), + with_node.get_column()) + if i == 1: + break + body = [wi] + return wi def handle_classdef(self, classdef_node, decorators=None): name_node = classdef_node.get_child(1) - name = self.new_identifier(name_node.get_value()) + name = name_node.get_value() self.check_forbidden_name(name, name_node) if classdef_node.num_children() == 4: - # class NAME ':' suite body = self.handle_suite(classdef_node.get_child(3)) - return ast.ClassDef(name, None, None, body, decorators, + return ast.ClassDef(name, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) if classdef_node.get_child(3).type == tokens.RPAR: - # class NAME '(' ')' ':' suite body = self.handle_suite(classdef_node.get_child(5)) - return ast.ClassDef(name, None, None, body, decorators, + return ast.ClassDef(name, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) - - # class NAME '(' arglist ')' ':' suite - # build up a fake Call node so we can extract its pieces - call_name = ast.Name(name, ast.Load, classdef_node.get_lineno(), - classdef_node.get_column()) - call = self.handle_call(classdef_node.get_child(3), call_name) + bases = self.handle_class_bases(classdef_node.get_child(3)) body = self.handle_suite(classdef_node.get_child(6)) - return ast.ClassDef( - name, call.args, call.keywords, - body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) + return ast.ClassDef(name, bases, body, decorators, classdef_node.get_lineno(), + classdef_node.get_column()) def handle_class_bases(self, bases_node): if bases_node.num_children() == 1: return [self.handle_expr(bases_node.get_child(0))] return self.get_expression_list(bases_node) - def handle_funcdef_impl(self, funcdef_node, is_async, decorators=None): + def handle_funcdef(self, funcdef_node, decorators=None): name_node = funcdef_node.get_child(1) - name = self.new_identifier(name_node.get_value()) + name = name_node.get_value() self.check_forbidden_name(name, name_node) args = self.handle_arguments(funcdef_node.get_child(2)) - suite = 4 - returns = None - if funcdef_node.get_child(3).type == tokens.RARROW: - returns = self.handle_expr(funcdef_node.get_child(4)) - suite += 2 - body = self.handle_suite(funcdef_node.get_child(suite)) - if is_async: - return ast.AsyncFunctionDef(name, args, body, decorators, returns, - funcdef_node.get_lineno(), funcdef_node.get_column()) - else: - return ast.FunctionDef(name, args, body, decorators, returns, - funcdef_node.get_lineno(), funcdef_node.get_column()) - - def handle_async_funcdef(self, node, decorators=None): - return self.handle_funcdef_impl(node.get_child(1), 1, decorators) - - def handle_funcdef(self, node, decorators=None): - return self.handle_funcdef_impl(node, 0, decorators) - - def handle_async_stmt(self, node): - ch = node.get_child(1) - if ch.type == syms.funcdef: - return self.handle_funcdef_impl(ch, 1) - elif ch.type == syms.with_stmt: - return self.handle_with_stmt(ch, 1) - elif ch.type == syms.for_stmt: - return self.handle_for_stmt(ch, 1) - else: - raise AssertionError("invalid async statement") + body = self.handle_suite(funcdef_node.get_child(4)) + return ast.FunctionDef(name, args, body, decorators, + funcdef_node.get_lineno(), funcdef_node.get_column()) def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) @@ -517,8 +499,6 @@ node = self.handle_funcdef(definition, decorators) elif definition.type == syms.classdef: node = self.handle_classdef(definition, decorators) - elif definition.type == syms.async_funcdef: - node = self.handle_async_funcdef(definition, decorators) else: raise AssertionError("unkown decorated") node.lineno = decorated_node.get_lineno() @@ -534,152 +514,110 @@ if decorator_node.num_children() == 3: dec = dec_name elif decorator_node.num_children() == 5: - dec = ast.Call(dec_name, None, None, + dec = ast.Call(dec_name, None, None, None, None, decorator_node.get_lineno(), decorator_node.get_column()) else: dec = self.handle_call(decorator_node.get_child(3), dec_name) return dec def handle_dotted_name(self, dotted_name_node): - base_value = self.new_identifier(dotted_name_node.get_child(0).get_value()) + base_value = dotted_name_node.get_child(0).get_value() name = ast.Name(base_value, ast.Load, dotted_name_node.get_lineno(), dotted_name_node.get_column()) for i in range(2, dotted_name_node.num_children(), 2): attr = dotted_name_node.get_child(i).get_value() - attr = self.new_identifier(attr) name = ast.Attribute(name, attr, ast.Load, dotted_name_node.get_lineno(), dotted_name_node.get_column()) return name def handle_arguments(self, arguments_node): - # This function handles both typedargslist (function definition) - # and varargslist (lambda definition). if arguments_node.type == syms.parameters: if arguments_node.num_children() == 2: - return ast.arguments(None, None, None, None, None, None) + return ast.arguments(None, None, None, None) arguments_node = arguments_node.get_child(1) i = 0 child_count = arguments_node.num_children() - n_pos = 0 - n_pos_def = 0 - n_kwdonly = 0 - # scan args - while i < child_count: - arg_type = arguments_node.get_child(i).type - if arg_type == tokens.STAR: - i += 1 - if i < child_count: - next_arg_type = arguments_node.get_child(i).type - if (next_arg_type == syms.tfpdef or - next_arg_type == syms.vfpdef): - i += 1 - break - if arg_type == tokens.DOUBLESTAR: - break - if arg_type == syms.vfpdef or arg_type == syms.tfpdef: - n_pos += 1 - if arg_type == tokens.EQUAL: - n_pos_def += 1 - i += 1 - while i < child_count: - arg_type = arguments_node.get_child(i).type - if arg_type == tokens.DOUBLESTAR: - break - if arg_type == syms.vfpdef or arg_type == syms.tfpdef: - n_kwdonly += 1 - i += 1 - pos = [] - posdefaults = [] - kwonly = [] if n_kwdonly else None - kwdefaults = [] - kwarg = None - vararg = None - if n_pos + n_kwdonly > 255: - self.error("more than 255 arguments", arguments_node) - # process args - i = 0 + defaults = [] + args = [] + variable_arg = None + keywords_arg = None have_default = False while i < child_count: - arg = arguments_node.get_child(i) - arg_type = arg.type - if arg_type == syms.tfpdef or arg_type == syms.vfpdef: - if i + 1 < child_count and \ - arguments_node.get_child(i + 1).type == tokens.EQUAL: - default_node = arguments_node.get_child(i + 2) - posdefaults.append(self.handle_expr(default_node)) + argument = arguments_node.get_child(i) + arg_type = argument.type + if arg_type == syms.fpdef: + parenthesized = False + complex_args = False + while True: + if i + 1 < child_count and \ + arguments_node.get_child(i + 1).type == tokens.EQUAL: + default_node = arguments_node.get_child(i + 2) + defaults.append(self.handle_expr(default_node)) + i += 2 + have_default = True + elif have_default: + if parenthesized and not complex_args: + msg = "parenthesized arg with default" + else: + msg = ("non-default argument follows default " + "argument") + self.error(msg, arguments_node) + if argument.num_children() == 3: + sub_arg = argument.get_child(1) + if sub_arg.num_children() != 1: + complex_args = True + args.append(self.handle_arg_unpacking(sub_arg)) + else: + parenthesized = True + argument = sub_arg.get_child(0) + continue + if argument.get_child(0).type == tokens.NAME: + name_node = argument.get_child(0) + arg_name = name_node.get_value() + self.check_forbidden_name(arg_name, name_node) + name = ast.Name(arg_name, ast.Param, name_node.get_lineno(), + name_node.get_column()) + args.append(name) i += 2 - have_default = True - elif have_default: - msg = "non-default argument follows default argument" - self.error(msg, arguments_node) - pos.append(self.handle_arg(arg)) - i += 2 + break elif arg_type == tokens.STAR: - if i + 1 >= child_count: - self.error("named arguments must follow bare *", - arguments_node) name_node = arguments_node.get_child(i + 1) - keywordonly_args = [] - if name_node.type == tokens.COMMA: - i += 2 - i = self.handle_keywordonly_args(arguments_node, i, kwonly, - kwdefaults) - else: - vararg = self.handle_arg(name_node) - i += 3 - if i < child_count: - next_arg_type = arguments_node.get_child(i).type - if (next_arg_type == syms.tfpdef or - next_arg_type == syms.vfpdef): - i = self.handle_keywordonly_args(arguments_node, i, - kwonly, kwdefaults) + variable_arg = name_node.get_value() + self.check_forbidden_name(variable_arg, name_node) + i += 3 elif arg_type == tokens.DOUBLESTAR: name_node = arguments_node.get_child(i + 1) - kwarg = self.handle_arg(name_node) + keywords_arg = name_node.get_value() + self.check_forbidden_name(keywords_arg, name_node) i += 3 else: raise AssertionError("unknown node in argument list") - return ast.arguments(pos, vararg, kwonly, kwdefaults, kwarg, - posdefaults) + if not defaults: + defaults = None + if not args: + args = None + return ast.arguments(args, variable_arg, keywords_arg, defaults) - def handle_keywordonly_args(self, arguments_node, i, kwonly, kwdefaults): - if kwonly is None: - self.error("named arguments must follows bare *", - arguments_node.get_child(i)) - child_count = arguments_node.num_children() - while i < child_count: - arg = arguments_node.get_child(i) - arg_type = arg.type - if arg_type == syms.vfpdef or arg_type == syms.tfpdef: - if (i + 1 < child_count and - arguments_node.get_child(i + 1).type == tokens.EQUAL): - expr = self.handle_expr(arguments_node.get_child(i + 2)) - kwdefaults.append(expr) - i += 2 + def handle_arg_unpacking(self, fplist_node): + args = [] + for i in range((fplist_node.num_children() + 1) / 2): + fpdef_node = fplist_node.get_child(i * 2) + while True: + child = fpdef_node.get_child(0) + if child.type == tokens.NAME: + arg = ast.Name(child.get_value(), ast.Store, child.get_lineno(), + child.get_column()) + args.append(arg) else: - kwdefaults.append(None) - ann = None - if arg.num_children() == 3: - ann = self.handle_expr(arg.get_child(2)) - name_node = arg.get_child(0) - argname = name_node.get_value() - argname = self.new_identifier(argname) - self.check_forbidden_name(argname, name_node) - kwonly.append(ast.arg(argname, ann, arg.get_lineno(), - arg.get_column())) - i += 2 - elif arg_type == tokens.DOUBLESTAR: - return i - return i - - def handle_arg(self, arg_node): - name_node = arg_node.get_child(0) - name = self.new_identifier(name_node.get_value()) - self.check_forbidden_name(name, arg_node) - ann = None - if arg_node.num_children() == 3: - ann = self.handle_expr(arg_node.get_child(2)) - return ast.arg(name, ann, arg_node.get_lineno(), arg_node.get_column()) + child = fpdef_node.get_child(1) + if child.num_children() == 1: + fpdef_node = child.get_child(0) + continue + args.append(self.handle_arg_unpacking(child)) + break + tup = ast.Tuple(args, ast.Store, fplist_node.get_lineno(), fplist_node.get_column()) + self.set_context(tup, ast.Store) + return tup def handle_stmt(self, stmt): stmt_type = stmt.type @@ -694,6 +632,8 @@ stmt_type = stmt.type if stmt_type == syms.expr_stmt: return self.handle_expr_stmt(stmt) + elif stmt_type == syms.print_stmt: + return self.handle_print_stmt(stmt) elif stmt_type == syms.del_stmt: return self.handle_del_stmt(stmt) elif stmt_type == syms.pass_stmt: @@ -704,10 +644,10 @@ return self.handle_import_stmt(stmt) elif stmt_type == syms.global_stmt: return self.handle_global_stmt(stmt) - elif stmt_type == syms.nonlocal_stmt: - return self.handle_nonlocal_stmt(stmt) elif stmt_type == syms.assert_stmt: return self.handle_assert_stmt(stmt) + elif stmt_type == syms.exec_stmt: + return self.handle_exec_stmt(stmt) else: raise AssertionError("unhandled small statement") elif stmt_type == syms.compound_stmt: @@ -718,19 +658,17 @@ elif stmt_type == syms.while_stmt: return self.handle_while_stmt(stmt) elif stmt_type == syms.for_stmt: - return self.handle_for_stmt(stmt, 0) + return self.handle_for_stmt(stmt) elif stmt_type == syms.try_stmt: return self.handle_try_stmt(stmt) elif stmt_type == syms.with_stmt: - return self.handle_with_stmt(stmt, 0) + return self.handle_with_stmt(stmt) elif stmt_type == syms.funcdef: return self.handle_funcdef(stmt) elif stmt_type == syms.classdef: return self.handle_classdef(stmt) elif stmt_type == syms.decorated: return self.handle_decorated(stmt) - elif stmt_type == syms.async_stmt: - return self.handle_async_stmt(stmt) else: raise AssertionError("unhandled compound statement") else: @@ -760,13 +698,12 @@ for i in range(0, stmt.num_children() - 2, 2): target_node = stmt.get_child(i) if target_node.type == syms.yield_expr: - self.error("assignment to yield expression not possible", - target_node) + self.error("can't assign to yield expression", target_node) target_expr = self.handle_testlist(target_node) self.set_context(target_expr, ast.Store) targets.append(target_expr) value_child = stmt.get_child(-1) - if value_child.type == syms.testlist_star_expr: + if value_child.type == syms.testlist: value_expr = self.handle_testlist(value_child) else: value_expr = self.handle_expr(value_child) @@ -787,9 +724,9 @@ # Loop until we return something. while True: expr_node_type = expr_node.type - if expr_node_type == syms.test or expr_node_type == syms.test_nocond: + if expr_node_type == syms.test or expr_node_type == syms.old_test: first_child = expr_node.get_child(0) - if first_child.type in (syms.lambdef, syms.lambdef_nocond): + if first_child.type in (syms.lambdef, syms.old_lambdef): return self.handle_lambdef(first_child) elif expr_node.num_children() > 1: return self.handle_ifexp(expr_node) @@ -826,8 +763,6 @@ operands.append(self.handle_expr(expr_node.get_child(i + 1))) return ast.Compare(expr, operators, operands, expr_node.get_lineno(), expr_node.get_column()) - elif expr_node_type == syms.star_expr: - return self.handle_star_expr(expr_node) elif expr_node_type == syms.expr or \ expr_node_type == syms.xor_expr or \ expr_node_type == syms.and_expr or \ @@ -839,19 +774,11 @@ continue return self.handle_binop(expr_node) elif expr_node_type == syms.yield_expr: - is_from = False - if expr_node.num_children() > 1: - arg_node = expr_node.get_child(1) # yield arg - if arg_node.num_children() == 2: - is_from = True - expr = self.handle_expr(arg_node.get_child(1)) - else: - expr = self.handle_testlist(arg_node.get_child(0)) + if expr_node.num_children() == 2: + exp = self.handle_testlist(expr_node.get_child(1)) else: - expr = None - if is_from: - return ast.YieldFrom(expr, expr_node.get_lineno(), expr_node.get_column()) - return ast.Yield(expr, expr_node.get_lineno(), expr_node.get_column()) + exp = None + return ast.Yield(exp, expr_node.get_lineno(), expr_node.get_column()) elif expr_node_type == syms.factor: if expr_node.num_children() == 1: expr_node = expr_node.get_child(0) @@ -862,15 +789,10 @@ else: raise AssertionError("unknown expr") - def handle_star_expr(self, star_expr_node): - expr = self.handle_expr(star_expr_node.get_child(1)) - return ast.Starred(expr, ast.Load, star_expr_node.get_lineno(), - star_expr_node.get_column()) - def handle_lambdef(self, lambdef_node): expr = self.handle_expr(lambdef_node.get_child(-1)) if lambdef_node.num_children() == 3: - args = ast.arguments(None, None, None, None, None, None) + args = ast.arguments(None, None, None, None) else: args = self.handle_arguments(lambdef_node.get_child(1)) return ast.Lambda(args, expr, lambdef_node.get_lineno(), lambdef_node.get_column()) @@ -897,11 +819,6 @@ elif comp_type == tokens.GREATEREQUAL: return ast.GtE elif comp_type == tokens.NOTEQUAL: - flufl = self.compile_info.flags & consts.CO_FUTURE_BARRY_AS_BDFL - if flufl and comp_node.get_value() == '!=': - self.error('invalid comparison', comp_node) - elif not flufl and comp_node.get_value() == '<>': - self.error('invalid comparison', comp_node) return ast.NotEq elif comp_type == tokens.NAME: if comp_node.get_value() == "is": @@ -937,6 +854,20 @@ def handle_factor(self, factor_node): from pypy.interpreter.pyparser.parser import Terminal + # Fold '-' on constant numbers. + if factor_node.get_child(0).type == tokens.MINUS and \ + factor_node.num_children() == 2: + factor = factor_node.get_child(1) + if factor.type == syms.factor and factor.num_children() == 1: + power = factor.get_child(0) + if power.type == syms.power and power.num_children() == 1: + atom = power.get_child(0) + if atom.type == syms.atom and \ + atom.get_child(0).type == tokens.NUMBER: + num = atom.get_child(0) + assert isinstance(num, Terminal) + num.value = "-" + num.get_value() + return self.handle_atom(atom) expr = self.handle_expr(factor_node.get_child(1)) op_type = factor_node.get_child(0).type if op_type == tokens.PLUS: @@ -949,35 +880,18 @@ raise AssertionError("invalid factor node") return ast.UnaryOp(op, expr, factor_node.get_lineno(), factor_node.get_column()) - def handle_atom_expr(self, atom_node): - start = 0 - num_ch = atom_node.num_children() - if atom_node.get_child(0).type == tokens.AWAIT: - start = 1 - atom_expr = self.handle_atom(atom_node.get_child(start)) - if num_ch == 1: + def handle_power(self, power_node): + atom_expr = self.handle_atom(power_node.get_child(0)) + if power_node.num_children() == 1: return atom_expr - if start and num_ch == 2: - return ast.Await(atom_expr, atom_node.get_lineno(), - atom_node.get_column()) - for i in range(start+1, num_ch): - trailer = atom_node.get_child(i) + for i in range(1, power_node.num_children()): + trailer = power_node.get_child(i) if trailer.type != syms.trailer: break tmp_atom_expr = self.handle_trailer(trailer, atom_expr) tmp_atom_expr.lineno = atom_expr.lineno tmp_atom_expr.col_offset = atom_expr.col_offset atom_expr = tmp_atom_expr - if start: - return ast.Await(atom_expr, atom_node.get_lineno(), - atom_node.get_column()) - else: - return atom_expr - - def handle_power(self, power_node): - atom_expr = self.handle_atom_expr(power_node.get_child(0)) - if power_node.num_children() == 1: - return atom_expr if power_node.get_child(-1).type == syms.factor: right = self.handle_expr(power_node.get_child(-1)) atom_expr = ast.BinOp(atom_expr, ast.Pow, right, power_node.get_lineno(), @@ -986,6 +900,8 @@ def handle_slice(self, slice_node): first_child = slice_node.get_child(0) + if first_child.type == tokens.DOT: + return ast.Ellipsis() if slice_node.num_children() == 1 and first_child.type == syms.test: index = self.handle_expr(first_child) return ast.Index(index) @@ -1005,7 +921,10 @@ upper = self.handle_expr(third_child) last_child = slice_node.get_child(-1) if last_child.type == syms.sliceop: - if last_child.num_children() != 1: + if last_child.num_children() == 1: + step = ast.Name("None", ast.Load, last_child.get_lineno(), + last_child.get_column()) + else: step_child = last_child.get_child(1) if step_child.type == syms.test: step = self.handle_expr(step_child) @@ -1015,12 +934,12 @@ first_child = trailer_node.get_child(0) if first_child.type == tokens.LPAR: if trailer_node.num_children() == 2: - return ast.Call(left_expr, None, None, + return ast.Call(left_expr, None, None, None, None, trailer_node.get_lineno(), trailer_node.get_column()) else: return self.handle_call(trailer_node.get_child(1), left_expr) elif first_child.type == tokens.DOT: - attr = self.new_identifier(trailer_node.get_child(1).get_value()) + attr = trailer_node.get_child(1).get_value() return ast.Attribute(left_expr, attr, ast.Load, trailer_node.get_lineno(), trailer_node.get_column()) else: @@ -1049,9 +968,9 @@ middle.get_lineno(), middle.get_column()) def handle_call(self, args_node, callable_expr): - arg_count = 0 # position args + iterable args unpackings - keyword_count = 0 # keyword args + keyword args unpackings - generator_count = 0 + arg_count = 0 + keyword_count = 0 + generator_count = 0 for i in range(args_node.num_children()): argument = args_node.get_child(i) if argument.type == syms.argument: @@ -1059,11 +978,7 @@ arg_count += 1 elif argument.get_child(1).type == syms.comp_for: generator_count += 1 - elif argument.get_child(0).type == tokens.STAR: - arg_count += 1 else: - # argument.get_child(0).type == tokens.DOUBLESTAR - # or keyword arg keyword_count += 1 if generator_count > 1 or \ (generator_count and (keyword_count or arg_count)): @@ -1074,66 +989,53 @@ args = [] keywords = [] used_keywords = {} - doublestars_count = 0 # just keyword argument unpackings + variable_arg = None + keywords_arg = None child_count = args_node.num_children() i = 0 while i < child_count: argument = args_node.get_child(i) if argument.type == syms.argument: - expr_node = argument.get_child(0) if argument.num_children() == 1: - # a positional argument + expr_node = argument.get_child(0) if keywords: - if doublestars_count: - self.error("positional argument follows " - "keyword argument unpacking", - expr_node) - else: - self.error("positional argument follows " - "keyword argument", - expr_node) + self.error("non-keyword arg after keyword arg", + expr_node) + if variable_arg: + self.error("only named arguments may follow " + "*expression", expr_node) args.append(self.handle_expr(expr_node)) - elif expr_node.type == tokens.STAR: - # an iterable argument unpacking - if doublestars_count: - self.error("iterable argument unpacking follows " - "keyword argument unpacking", - expr_node) - expr = self.handle_expr(argument.get_child(1)) - args.append(ast.Starred(expr, ast.Load, - expr_node.get_lineno(), - expr_node.get_column())) - elif expr_node.type == tokens.DOUBLESTAR: - # a keyword argument unpacking - i += 1 - expr = self.handle_expr(argument.get_child(1)) - keywords.append(ast.keyword(None, expr)) - doublestars_count += 1 elif argument.get_child(1).type == syms.comp_for: - # the lone generator expression args.append(self.handle_genexp(argument)) else: - # a keyword argument - keyword_expr = self.handle_expr(expr_node) + keyword_node = argument.get_child(0) + keyword_expr = self.handle_expr(keyword_node) if isinstance(keyword_expr, ast.Lambda): self.error("lambda cannot contain assignment", - expr_node) + keyword_node) elif not isinstance(keyword_expr, ast.Name): self.error("keyword can't be an expression", - expr_node) + keyword_node) keyword = keyword_expr.id if keyword in used_keywords: - self.error("keyword argument repeated", expr_node) + self.error("keyword argument repeated", keyword_node) used_keywords[keyword] = None - self.check_forbidden_name(keyword, expr_node) + self.check_forbidden_name(keyword, keyword_node) keyword_value = self.handle_expr(argument.get_child(2)) keywords.append(ast.keyword(keyword, keyword_value)) + elif argument.type == tokens.STAR: + variable_arg = self.handle_expr(args_node.get_child(i + 1)) + i += 1 + elif argument.type == tokens.DOUBLESTAR: + keywords_arg = self.handle_expr(args_node.get_child(i + 1)) + i += 1 i += 1 if not args: args = None if not keywords: keywords = None - return ast.Call(callable_expr, args, keywords, callable_expr.lineno, + return ast.Call(callable_expr, args, keywords, variable_arg, + keywords_arg, callable_expr.lineno, callable_expr.col_offset) def parse_number(self, raw): @@ -1170,7 +1072,10 @@ raw = "-" + raw w_num_str = self.space.newtext(raw) w_base = self.space.newint(base) - if raw[-1] in "jJ": + if raw[-1] in "lL": + tp = self.space.w_long + return self.space.call_function(tp, w_num_str, w_base) + elif raw[-1] in "jJ": tp = self.space.w_complex return self.space.call_function(tp, w_num_str) try: @@ -1180,44 +1085,43 @@ raise return self.space.call_function(self.space.w_float, w_num_str) - @always_inline - def handle_dictelement(self, node, i): - if node.get_child(i).type == tokens.DOUBLESTAR: - key = None - value = self.handle_expr(node.get_child(i+1)) - i += 2 - else: - key = self.handle_expr(node.get_child(i)) - value = self.handle_expr(node.get_child(i+2)) - i += 3 - return (i,key,value) - def handle_atom(self, atom_node): first_child = atom_node.get_child(0) first_child_type = first_child.type if first_child_type == tokens.NAME: - name = first_child.get_value() - if name == "None": - w_singleton = self.space.w_None - elif name == "True": - w_singleton = self.space.w_True - elif name == "False": - w_singleton = self.space.w_False + return ast.Name(first_child.get_value(), ast.Load, + first_child.get_lineno(), first_child.get_column()) + elif first_child_type == tokens.STRING: + space = self.space + encoding = self.compile_info.encoding + flags = self.compile_info.flags + unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), + unicode_literals)) + except error.OperationError as e: + if not e.match(space, space.w_UnicodeError): + raise + # UnicodeError in literal: turn into SyntaxError + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + if child is None: + child = atom_node + raise self.error('(unicode error) %s' % errmsg, child) + # This implements implicit string concatenation. + if len(sub_strings_w) > 1: + w_sub_strings = space.newlist(sub_strings_w) + w_join = space.getattr(space.newtext(""), space.newtext("join")) + final_string = space.call_function(w_join, w_sub_strings) else: - name = self.new_identifier(name) - return ast.Name(name, ast.Load, first_child.get_lineno(), - first_child.get_column()) - return ast.NameConstant(w_singleton, first_child.get_lineno(), - first_child.get_column()) - # - elif first_child_type == tokens.STRING: - return fstring.string_parse_literal(self, atom_node) - # + final_string = sub_strings_w[0] + return ast.Str(final_string, atom_node.get_lineno(), atom_node.get_column()) elif first_child_type == tokens.NUMBER: num_value = self.parse_number(first_child.get_value()) return ast.Num(num_value, atom_node.get_lineno(), atom_node.get_column()) - elif first_child_type == tokens.ELLIPSIS: - return ast.Ellipsis(atom_node.get_lineno(), atom_node.get_column()) elif first_child_type == tokens.LPAR: second_child = atom_node.get_child(1) if second_child.type == tokens.RPAR: @@ -1239,31 +1143,28 @@ return self.handle_listcomp(second_child) elif first_child_type == tokens.LBRACE: maker = atom_node.get_child(1) + if maker.type == tokens.RBRACE: + return ast.Dict(None, None, atom_node.get_lineno(), atom_node.get_column()) n_maker_children = maker.num_children() - if maker.type == tokens.RBRACE: - # an empty dict - return ast.Dict(None, None, atom_node.get_lineno(), atom_node.get_column()) - else: - is_dict = maker.get_child(0).type == tokens.DOUBLESTAR - if (n_maker_children == 1 or - (n_maker_children > 1 and - maker.get_child(1).type == tokens.COMMA)): - # a set display - return self.handle_setdisplay(maker, atom_node) - elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for: - # a set comprehension - return self.handle_setcomp(maker, atom_node) - elif (n_maker_children > (3-is_dict) and - maker.get_child(3-is_dict).type == syms.comp_for): - # a dictionary comprehension - if is_dict: - raise self.error("dict unpacking cannot be used in " - "dict comprehension", atom_node) - - return self.handle_dictcomp(maker, atom_node) - else: - # a dictionary display - return self.handle_dictdisplay(maker, atom_node) + if n_maker_children == 1 or maker.get_child(1).type == tokens.COMMA: + elts = [] + for i in range(0, n_maker_children, 2): + elts.append(self.handle_expr(maker.get_child(i))) + return ast.Set(elts, atom_node.get_lineno(), atom_node.get_column()) + if maker.get_child(1).type == syms.comp_for: + return self.handle_setcomp(maker) + if (n_maker_children > 3 and + maker.get_child(3).type == syms.comp_for): + return self.handle_dictcomp(maker) + keys = [] + values = [] + for i in range(0, n_maker_children, 4): + keys.append(self.handle_expr(maker.get_child(i))) + values.append(self.handle_expr(maker.get_child(i + 2))) + return ast.Dict(keys, values, atom_node.get_lineno(), atom_node.get_column()) + elif first_child_type == tokens.BACKQUOTE: + expr = self.handle_testlist(atom_node.get_child(1)) + return ast.Repr(expr, atom_node.get_lineno(), atom_node.get_column()) else: raise AssertionError("unknown atom") @@ -1273,7 +1174,7 @@ return self.handle_genexp(gexp_node) return self.handle_testlist(gexp_node) - def count_comp_fors(self, comp_node): + def count_comp_fors(self, comp_node, for_type, if_type): count = 0 current_for = comp_node while True: @@ -1284,10 +1185,10 @@ return count while True: first_child = current_iter.get_child(0) - if first_child.type == syms.comp_for: + if first_child.type == for_type: current_for = current_iter.get_child(0) break - elif first_child.type == syms.comp_if: + elif first_child.type == if_type: if first_child.num_children() == 3: current_iter = first_child.get_child(2) else: @@ -1295,40 +1196,48 @@ else: raise AssertionError("should not reach here") - def count_comp_ifs(self, iter_node): + def count_comp_ifs(self, iter_node, for_type): count = 0 while True: first_child = iter_node.get_child(0) - if first_child.type == syms.comp_for: + if first_child.type == for_type: return count count += 1 if first_child.num_children() == 2: return count iter_node = first_child.get_child(2) - def comprehension_helper(self, comp_node): - fors_count = self.count_comp_fors(comp_node) + @specialize.arg(2) + def comprehension_helper(self, comp_node, + handle_source_expr_meth="handle_expr", + for_type=syms.comp_for, if_type=syms.comp_if, + iter_type=syms.comp_iter, + comp_fix_unamed_tuple_location=False): + handle_source_expression = getattr(self, handle_source_expr_meth) + fors_count = self.count_comp_fors(comp_node, for_type, if_type) comps = [] for i in range(fors_count): for_node = comp_node.get_child(1) for_targets = self.handle_exprlist(for_node, ast.Store) - expr = self.handle_expr(comp_node.get_child(3)) + expr = handle_source_expression(comp_node.get_child(3)) assert isinstance(expr, ast.expr) if for_node.num_children() == 1: comp = ast.comprehension(for_targets[0], expr, None) else: + col = comp_node.get_column() + line = comp_node.get_lineno() # Modified in python2.7, see http://bugs.python.org/issue6704 - # Fixing unamed tuple location - expr_node = for_targets[0] - assert isinstance(expr_node, ast.expr) - col = expr_node.col_offset - line = expr_node.lineno + if comp_fix_unamed_tuple_location: + expr_node = for_targets[0] + assert isinstance(expr_node, ast.expr) + col = expr_node.col_offset + line = expr_node.lineno target = ast.Tuple(for_targets, ast.Store, line, col) comp = ast.comprehension(target, expr, None) if comp_node.num_children() == 5: comp_node = comp_iter = comp_node.get_child(4) - assert comp_iter.type == syms.comp_iter - ifs_count = self.count_comp_ifs(comp_iter) + assert comp_iter.type == iter_type + ifs_count = self.count_comp_ifs(comp_iter, for_type) if ifs_count: ifs = [] for j in range(ifs_count): @@ -1337,66 +1246,42 @@ if comp_if.num_children() == 3: comp_node = comp_iter = comp_if.get_child(2) comp.ifs = ifs - if comp_node.type == syms.comp_iter: + if comp_node.type == iter_type: comp_node = comp_node.get_child(0) assert isinstance(comp, ast.comprehension) comps.append(comp) return comps def handle_genexp(self, genexp_node): - ch = genexp_node.get_child(0) - elt = self.handle_expr(ch) - if isinstance(elt, ast.Starred): - self.error("iterable unpacking cannot be used in comprehension", ch) - comps = self.comprehension_helper(genexp_node.get_child(1)) + elt = self.handle_expr(genexp_node.get_child(0)) + comps = self.comprehension_helper(genexp_node.get_child(1), + comp_fix_unamed_tuple_location=True) return ast.GeneratorExp(elt, comps, genexp_node.get_lineno(), genexp_node.get_column()) def handle_listcomp(self, listcomp_node): - ch = listcomp_node.get_child(0) - elt = self.handle_expr(ch) - if isinstance(elt, ast.Starred): - self.error("iterable unpacking cannot be used in comprehension", ch) - comps = self.comprehension_helper(listcomp_node.get_child(1)) + elt = self.handle_expr(listcomp_node.get_child(0)) + comps = self.comprehension_helper(listcomp_node.get_child(1), + "handle_testlist", + syms.list_for, syms.list_if, + syms.list_iter, + comp_fix_unamed_tuple_location=True) return ast.ListComp(elt, comps, listcomp_node.get_lineno(), listcomp_node.get_column()) - def handle_setcomp(self, set_maker, atom_node): - ch = set_maker.get_child(0) - elt = self.handle_expr(ch) - if isinstance(elt, ast.Starred): - self.error("iterable unpacking cannot be used in comprehension", ch) - comps = self.comprehension_helper(set_maker.get_child(1)) - return ast.SetComp(elt, comps, atom_node.get_lineno(), - atom_node.get_column()) + def handle_setcomp(self, set_maker): + elt = self.handle_expr(set_maker.get_child(0)) + comps = self.comprehension_helper(set_maker.get_child(1), + comp_fix_unamed_tuple_location=True) + return ast.SetComp(elt, comps, set_maker.get_lineno(), set_maker.get_column()) - def handle_dictcomp(self, dict_maker, atom_node): - i, key, value = self.handle_dictelement(dict_maker, 0) - comps = self.comprehension_helper(dict_maker.get_child(i)) - return ast.DictComp(key, value, comps, atom_node.get_lineno(), - atom_node.get_column()) - - def handle_dictdisplay(self, node, atom_node): - keys = [] - values = [] - i = 0 - while i < node.num_children(): - i, key, value = self.handle_dictelement(node, i) - keys.append(key) - values.append(value) - i += 1 - return ast.Dict(keys, values, atom_node.get_lineno(), - atom_node.get_column()) - - def handle_setdisplay(self, node, atom_node): - elts = [] - i = 0 - while i < node.num_children(): - expr = self.handle_expr(node.get_child(i)) - elts.append(expr) - i += 2 - return ast.Set(elts, atom_node.get_lineno(), - atom_node.get_column()) + def handle_dictcomp(self, dict_maker): + key = self.handle_expr(dict_maker.get_child(0)) + value = self.handle_expr(dict_maker.get_child(2)) + comps = self.comprehension_helper(dict_maker.get_child(3), + comp_fix_unamed_tuple_location=True) + return ast.DictComp(key, value, comps, dict_maker.get_lineno(), + dict_maker.get_column()) def handle_exprlist(self, exprlist, context): exprs = [] diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1408,3 +1408,11 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -207,6 +207,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -134,7 +134,7 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.3" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h --- a/pypy/module/cpyext/include/longobject.h +++ b/pypy/module/cpyext/include/longobject.h @@ -20,6 +20,9 @@ #define PyLong_AS_LONG(op) PyLong_AsLong(op) +#define _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) \ + _PyLong_AsByteArrayO((PyObject *)(v), bytes, n, little_endian, is_signed) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -3,8 +3,8 @@ cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t, CONST_STRING, ADDR, CANNOT_FAIL) from pypy.objspace.std.longobject import W_LongObject -from pypy.interpreter.error import OperationError -from rpython.rlib.rbigint import rbigint +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rbigint import rbigint, InvalidSignednessError PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long") @@ -251,3 +251,26 @@ byteorder = 'big' result = rbigint.frombytes(s, byteorder, signed != 0) return space.newlong_from_rbigint(result) + + at cpython_api([PyObject, rffi.UCHARP, rffi.SIZE_T, + rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1) +def _PyLong_AsByteArrayO(space, w_v, bytes, n, little_endian, is_signed): + n = rffi.cast(lltype.Signed, n) + little_endian = rffi.cast(lltype.Signed, little_endian) + signed = rffi.cast(lltype.Signed, is_signed) != 0 + byteorder = 'little' if little_endian else 'big' + bigint = space.bigint_w(w_v) + try: + digits = bigint.tobytes(n, byteorder, signed) + except InvalidSignednessError: # < 0 but not 'signed' + # in this case, CPython raises OverflowError even though the C + # comments say it should raise TypeError + raise oefmt(space.w_OverflowError, + "can't convert negative long to unsigned") + except OverflowError: + raise oefmt(space.w_OverflowError, + "long too big to convert") + assert len(digits) == n + for i in range(n): + bytes[i] = rffi.cast(rffi.UCHAR, digits[i]) + return 0 diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -267,7 +267,7 @@ raise oefmt(space.w_ValueError, "sequence.index(x): x not in sequence") class CPyListStrategy(ListStrategy): - erase, unerase = rerased.new_erasing_pair("empty") + erase, unerase = rerased.new_erasing_pair("cpylist") erase = staticmethod(erase) unerase = staticmethod(unerase) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1577,13 +1577,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF32String(space, unicode): - """Return a Python byte string using the UTF-32 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If @@ -1612,13 +1605,6 @@ Return NULL if an exception was raised by the codec.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF16String(space, unicode): - """Return a Python byte string using the UTF-16 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_DecodeUTF7(space, s, size, errors): """Create a Unicode object by decoding size bytes of the UTF-7 encoded string diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -259,6 +259,48 @@ assert module.from_bytearray(False, False) == 0x9ABC41 assert module.from_bytearray(False, True) == -0x6543BF + def test_asbytearray(self): + module = self.import_extension('foo', [ + ("as_bytearray", "METH_VARARGS", + """ + PyObject *result; + PyLongObject *o; + int n, little_endian, is_signed; + unsigned char *bytes; + if (!PyArg_ParseTuple(args, "O!iii", &PyLong_Type, &o, &n, + &little_endian, &is_signed)) + return NULL; + bytes = malloc(n); + if (_PyLong_AsByteArray(o, bytes, (size_t)n, + little_endian, is_signed) != 0) + { + free(bytes); + return NULL; + } + result = PyString_FromStringAndSize((const char *)bytes, n); + free(bytes); + return result; + """), + ]) + s = module.as_bytearray(0x41BC9AL, 4, True, False) + assert s == "\x9A\xBC\x41\x00" + s = module.as_bytearray(0x41BC9AL, 4, False, False) + assert s == "\x00\x41\xBC\x9A" + s = module.as_bytearray(0x41BC9AL, 3, True, False) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x41BC9AL, 3, True, True) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x9876L, 2, True, False) + assert s == "\x76\x98" + s = module.as_bytearray(0x9876L - 0x10000L, 2, True, True) + assert s == "\x76\x98" + raises(OverflowError, module.as_bytearray, + 0x9876L, 2, False, True) + raises(OverflowError, module.as_bytearray, + -1L, 2, True, False) + raises(OverflowError, module.as_bytearray, + 0x1234567L, 3, True, False) + def test_fromunicode(self): module = self.import_extension('foo', [ ("from_unicode", "METH_O", diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -218,7 +218,7 @@ if not cls.runappdirect: cls.total_mem = 0 - def add_memory_pressure(estimate): + def add_memory_pressure(estimate, object=None): assert estimate >= 0 cls.total_mem += estimate cls.orig_add_memory_pressure = [rgc.add_memory_pressure] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,13 +3,23 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr import sys import pytest class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -162,6 +172,25 @@ assert fuu2(u"abc").baz().escape() From pypy.commits at gmail.com Fri Feb 9 09:12:23 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 06:12:23 -0800 (PST) Subject: [pypy-commit] pypy py3.5: took wrong parent in merge Message-ID: <5a7dac47.8583df0a.e7cbe.03f3@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93791:b5928bf4b1e3 Date: 2018-02-09 09:11 -0500 http://bitbucket.org/pypy/pypy/changeset/b5928bf4b1e3/ Log: took wrong parent in merge diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1,15 +1,23 @@ from pypy.interpreter.astcompiler import ast, consts, misc from pypy.interpreter.astcompiler import asthelpers # Side effects +from pypy.interpreter.astcompiler import fstring from pypy.interpreter import error from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError -from pypy.interpreter.pyparser import parsestring -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import always_inline, we_are_translated -def ast_from_node(space, node, compile_info): +def ast_from_node(space, node, compile_info, recursive_parser=None): """Turn a parse tree, node, to AST.""" - return ASTBuilder(space, node, compile_info).build_ast() + ast = ASTBuilder(space, node, compile_info, recursive_parser).build_ast() + # + # When we are not translated, we send this ast to validate_ast. + # The goal is to check that validate_ast doesn't crash on valid + # asts, at least. + if not we_are_translated(): + from pypy.interpreter.astcompiler import validate + validate.validate_ast(space, ast) + return ast augassign_operator_map = { @@ -18,6 +26,7 @@ '/=' : ast.Div, '//=' : ast.FloorDiv, '%=' : ast.Mod, + '@=' : ast.MatMult, '<<=' : ast.LShift, '>>=' : ast.RShift, '&=' : ast.BitAnd, @@ -38,16 +47,18 @@ tokens.STAR : ast.Mult, tokens.SLASH : ast.Div, tokens.DOUBLESLASH : ast.FloorDiv, - tokens.PERCENT : ast.Mod + tokens.PERCENT : ast.Mod, + tokens.AT : ast.MatMult }) class ASTBuilder(object): - def __init__(self, space, n, compile_info): + def __init__(self, space, n, compile_info, recursive_parser=None): self.space = space self.compile_info = compile_info self.root_node = n + self.recursive_parser = recursive_parser def build_ast(self): """Convert an top level parse tree node into an AST mod.""" @@ -118,6 +129,9 @@ except misc.ForbiddenNameAssignment as e: self.error("cannot assign to %s" % (e.name,), node) + def new_identifier(self, name): + return misc.new_identifier(self.space, name) + def set_context(self, expr, ctx): """Set the context of an expression to Store or Del if possible.""" try: @@ -127,23 +141,6 @@ except misc.ForbiddenNameAssignment as e: self.error_ast("cannot assign to %s" % (e.name,), e.node) - def handle_print_stmt(self, print_node): - dest = None - expressions = None - newline = True - start = 1 - child_count = print_node.num_children() - if child_count > 2 and print_node.get_child(1).type == tokens.RIGHTSHIFT: - dest = self.handle_expr(print_node.get_child(2)) - start = 4 - if (child_count + 1 - start) // 2: - expressions = [self.handle_expr(print_node.get_child(i)) - for i in range(start, child_count, 2)] - if print_node.get_child(-1).type == tokens.COMMA: - newline = False - return ast.Print(dest, expressions, newline, print_node.get_lineno(), - print_node.get_column()) - def handle_del_stmt(self, del_node): targets = self.handle_exprlist(del_node.get_child(1), ast.Del) return ast.Delete(targets, del_node.get_lineno(), del_node.get_column()) @@ -166,17 +163,13 @@ return ast.Return(values, flow_node.get_lineno(), flow_node.get_column()) elif first_child_type == syms.raise_stmt: exc = None - value = None - traceback = None + cause = None child_count = first_child.num_children() if child_count >= 2: exc = self.handle_expr(first_child.get_child(1)) if child_count >= 4: - value = self.handle_expr(first_child.get_child(3)) - if child_count == 6: - traceback = self.handle_expr(first_child.get_child(5)) - return ast.Raise(exc, value, traceback, flow_node.get_lineno(), - flow_node.get_column()) + cause = self.handle_expr(first_child.get_child(3)) + return ast.Raise(exc, cause, flow_node.get_lineno(), flow_node.get_column()) else: raise AssertionError("unknown flow statement") @@ -184,9 +177,10 @@ while True: import_name_type = import_name.type if import_name_type == syms.import_as_name: - name = import_name.get_child(0).get_value() + name = self.new_identifier(import_name.get_child(0).get_value()) if import_name.num_children() == 3: - as_name = import_name.get_child(2).get_value() + as_name = self.new_identifier( + import_name.get_child(2).get_value()) self.check_forbidden_name(as_name, import_name.get_child(2)) else: as_name = None @@ -199,12 +193,12 @@ alias = self.alias_for_import_name(import_name.get_child(0), store=False) asname_node = import_name.get_child(2) - alias.asname = asname_node.get_value() + alias.asname = self.new_identifier(asname_node.get_value()) self.check_forbidden_name(alias.asname, asname_node) return alias elif import_name_type == syms.dotted_name: if import_name.num_children() == 1: - name = import_name.get_child(0).get_value() + name = self.new_identifier(import_name.get_child(0).get_value()) if store: self.check_forbidden_name(name, import_name.get_child(0)) return ast.alias(name, None) @@ -232,11 +226,15 @@ dot_count = 0 while i < child_count: child = import_node.get_child(i) - if child.type == syms.dotted_name: + child_type = child.type + if child_type == syms.dotted_name: module = self.alias_for_import_name(child, False) i += 1 break - elif child.type != tokens.DOT: + elif child_type == tokens.ELLIPSIS: + # Special case for tokenization. + dot_count += 2 + elif child_type != tokens.DOT: break i += 1 dot_count += 1 @@ -268,28 +266,14 @@ raise AssertionError("unknown import node") def handle_global_stmt(self, global_node): - names = [global_node.get_child(i).get_value() + names = [self.new_identifier(global_node.get_child(i).get_value()) for i in range(1, global_node.num_children(), 2)] return ast.Global(names, global_node.get_lineno(), global_node.get_column()) - def handle_exec_stmt(self, exec_node): - child_count = exec_node.num_children() - globs = None - locs = None - to_execute = self.handle_expr(exec_node.get_child(1)) - if child_count < 4: - if isinstance(to_execute, ast.Tuple) and \ - (len(to_execute.elts) == 2 or len(to_execute.elts) == 3): - globs = to_execute.elts[1] - if len(to_execute.elts) == 3: - locs = to_execute.elts[2] - to_execute = to_execute.elts[0] - elif child_count >= 4: - globs = self.handle_expr(exec_node.get_child(3)) - if child_count == 6: - locs = self.handle_expr(exec_node.get_child(5)) - return ast.Exec(to_execute, globs, locs, exec_node.get_lineno(), - exec_node.get_column()) + def handle_nonlocal_stmt(self, nonlocal_node): + names = [self.new_identifier(nonlocal_node.get_child(i).get_value()) + for i in range(1, nonlocal_node.num_children(), 2)] + return ast.Nonlocal(names, nonlocal_node.get_lineno(), nonlocal_node.get_column()) def handle_assert_stmt(self, assert_node): expr = self.handle_expr(assert_node.get_child(1)) @@ -379,7 +363,7 @@ return ast.While(loop_test, body, otherwise, while_node.get_lineno(), while_node.get_column()) - def handle_for_stmt(self, for_node): + def handle_for_stmt(self, for_node, is_async): target_node = for_node.get_child(1) target_as_exprlist = self.handle_exprlist(target_node, ast.Store) if target_node.num_children() == 1: @@ -393,21 +377,25 @@ otherwise = self.handle_suite(for_node.get_child(8)) else: otherwise = None - return ast.For(target, expr, body, otherwise, for_node.get_lineno(), - for_node.get_column()) + if is_async: + return ast.AsyncFor(target, expr, body, otherwise, for_node.get_lineno(), + for_node.get_column()) + else: + return ast.For(target, expr, body, otherwise, for_node.get_lineno(), + for_node.get_column()) def handle_except_clause(self, exc, body): test = None - target = None + name = None suite = self.handle_suite(body) child_count = exc.num_children() if child_count >= 2: test = self.handle_expr(exc.get_child(1)) if child_count == 4: - target_child = exc.get_child(3) - target = self.handle_expr(target_child) - self.set_context(target, ast.Store) - return ast.ExceptHandler(test, target, suite, exc.get_lineno(), exc.get_column()) + name_node = exc.get_child(3) + name = self.new_identifier(name_node.get_value()) + self.check_forbidden_name(name, name_node) + return ast.ExceptHandler(test, name, suite, exc.get_lineno(), exc.get_column()) def handle_try_stmt(self, try_node): body = self.handle_suite(try_node.get_child(2)) @@ -427,70 +415,100 @@ else: otherwise = self.handle_suite(try_node.get_child(-1)) except_count -= 1 + handlers = [] if except_count: - handlers = [] for i in range(except_count): base_offset = i * 3 exc = try_node.get_child(3 + base_offset) except_body = try_node.get_child(5 + base_offset) handlers.append(self.handle_except_clause(exc, except_body)) - except_ast = ast.TryExcept(body, handlers, otherwise, - try_node.get_lineno(), try_node.get_column()) - if finally_suite is None: - return except_ast - body = [except_ast] - return ast.TryFinally(body, finally_suite, try_node.get_lineno(), - try_node.get_column()) + return ast.Try(body, handlers, otherwise, finally_suite, + try_node.get_lineno(), try_node.get_column()) - def handle_with_stmt(self, with_node): + def handle_with_item(self, item_node): + test = self.handle_expr(item_node.get_child(0)) + if item_node.num_children() == 3: + target = self.handle_expr(item_node.get_child(2)) + self.set_context(target, ast.Store) + else: + target = None + return ast.withitem(test, target) + + def handle_with_stmt(self, with_node, is_async): body = self.handle_suite(with_node.get_child(-1)) - i = with_node.num_children() - 1 - while True: - i -= 2 - item = with_node.get_child(i) - test = self.handle_expr(item.get_child(0)) - if item.num_children() == 3: - target = self.handle_expr(item.get_child(2)) - self.set_context(target, ast.Store) - else: - target = None - wi = ast.With(test, target, body, with_node.get_lineno(), - with_node.get_column()) - if i == 1: - break - body = [wi] - return wi + items = [self.handle_with_item(with_node.get_child(i)) + for i in range(1, with_node.num_children()-2, 2)] + if is_async: + return ast.AsyncWith(items, body, with_node.get_lineno(), + with_node.get_column()) + else: + return ast.With(items, body, with_node.get_lineno(), + with_node.get_column()) def handle_classdef(self, classdef_node, decorators=None): name_node = classdef_node.get_child(1) - name = name_node.get_value() + name = self.new_identifier(name_node.get_value()) self.check_forbidden_name(name, name_node) if classdef_node.num_children() == 4: + # class NAME ':' suite body = self.handle_suite(classdef_node.get_child(3)) - return ast.ClassDef(name, None, body, decorators, + return ast.ClassDef(name, None, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) if classdef_node.get_child(3).type == tokens.RPAR: + # class NAME '(' ')' ':' suite body = self.handle_suite(classdef_node.get_child(5)) - return ast.ClassDef(name, None, body, decorators, + return ast.ClassDef(name, None, None, body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) - bases = self.handle_class_bases(classdef_node.get_child(3)) + + # class NAME '(' arglist ')' ':' suite + # build up a fake Call node so we can extract its pieces + call_name = ast.Name(name, ast.Load, classdef_node.get_lineno(), + classdef_node.get_column()) + call = self.handle_call(classdef_node.get_child(3), call_name) body = self.handle_suite(classdef_node.get_child(6)) - return ast.ClassDef(name, bases, body, decorators, classdef_node.get_lineno(), - classdef_node.get_column()) + return ast.ClassDef( + name, call.args, call.keywords, + body, decorators, classdef_node.get_lineno(), classdef_node.get_column()) def handle_class_bases(self, bases_node): if bases_node.num_children() == 1: return [self.handle_expr(bases_node.get_child(0))] return self.get_expression_list(bases_node) - def handle_funcdef(self, funcdef_node, decorators=None): + def handle_funcdef_impl(self, funcdef_node, is_async, decorators=None): name_node = funcdef_node.get_child(1) - name = name_node.get_value() + name = self.new_identifier(name_node.get_value()) self.check_forbidden_name(name, name_node) args = self.handle_arguments(funcdef_node.get_child(2)) - body = self.handle_suite(funcdef_node.get_child(4)) - return ast.FunctionDef(name, args, body, decorators, - funcdef_node.get_lineno(), funcdef_node.get_column()) + suite = 4 + returns = None + if funcdef_node.get_child(3).type == tokens.RARROW: + returns = self.handle_expr(funcdef_node.get_child(4)) + suite += 2 + body = self.handle_suite(funcdef_node.get_child(suite)) + if is_async: + return ast.AsyncFunctionDef(name, args, body, decorators, returns, + funcdef_node.get_lineno(), funcdef_node.get_column()) + else: + return ast.FunctionDef(name, args, body, decorators, returns, + funcdef_node.get_lineno(), funcdef_node.get_column()) + + def handle_async_funcdef(self, node, decorators=None): + return self.handle_funcdef_impl(node.get_child(1), 1, decorators) + + def handle_funcdef(self, node, decorators=None): + return self.handle_funcdef_impl(node, 0, decorators) + + def handle_async_stmt(self, node): + ch = node.get_child(1) + if ch.type == syms.funcdef: + return self.handle_funcdef_impl(ch, 1) + elif ch.type == syms.with_stmt: + return self.handle_with_stmt(ch, 1) + elif ch.type == syms.for_stmt: + return self.handle_for_stmt(ch, 1) + else: + raise AssertionError("invalid async statement") def handle_decorated(self, decorated_node): decorators = self.handle_decorators(decorated_node.get_child(0)) @@ -499,6 +517,8 @@ node = self.handle_funcdef(definition, decorators) elif definition.type == syms.classdef: node = self.handle_classdef(definition, decorators) + elif definition.type == syms.async_funcdef: + node = self.handle_async_funcdef(definition, decorators) else: raise AssertionError("unkown decorated") node.lineno = decorated_node.get_lineno() @@ -514,110 +534,152 @@ if decorator_node.num_children() == 3: dec = dec_name elif decorator_node.num_children() == 5: - dec = ast.Call(dec_name, None, None, None, None, + dec = ast.Call(dec_name, None, None, decorator_node.get_lineno(), decorator_node.get_column()) else: dec = self.handle_call(decorator_node.get_child(3), dec_name) return dec def handle_dotted_name(self, dotted_name_node): - base_value = dotted_name_node.get_child(0).get_value() + base_value = self.new_identifier(dotted_name_node.get_child(0).get_value()) name = ast.Name(base_value, ast.Load, dotted_name_node.get_lineno(), dotted_name_node.get_column()) for i in range(2, dotted_name_node.num_children(), 2): attr = dotted_name_node.get_child(i).get_value() + attr = self.new_identifier(attr) name = ast.Attribute(name, attr, ast.Load, dotted_name_node.get_lineno(), dotted_name_node.get_column()) return name def handle_arguments(self, arguments_node): + # This function handles both typedargslist (function definition) + # and varargslist (lambda definition). if arguments_node.type == syms.parameters: if arguments_node.num_children() == 2: - return ast.arguments(None, None, None, None) + return ast.arguments(None, None, None, None, None, None) arguments_node = arguments_node.get_child(1) i = 0 child_count = arguments_node.num_children() - defaults = [] - args = [] - variable_arg = None - keywords_arg = None + n_pos = 0 + n_pos_def = 0 + n_kwdonly = 0 + # scan args + while i < child_count: + arg_type = arguments_node.get_child(i).type + if arg_type == tokens.STAR: + i += 1 + if i < child_count: + next_arg_type = arguments_node.get_child(i).type + if (next_arg_type == syms.tfpdef or + next_arg_type == syms.vfpdef): + i += 1 + break + if arg_type == tokens.DOUBLESTAR: + break + if arg_type == syms.vfpdef or arg_type == syms.tfpdef: + n_pos += 1 + if arg_type == tokens.EQUAL: + n_pos_def += 1 + i += 1 + while i < child_count: + arg_type = arguments_node.get_child(i).type + if arg_type == tokens.DOUBLESTAR: + break + if arg_type == syms.vfpdef or arg_type == syms.tfpdef: + n_kwdonly += 1 + i += 1 + pos = [] + posdefaults = [] + kwonly = [] if n_kwdonly else None + kwdefaults = [] + kwarg = None + vararg = None + if n_pos + n_kwdonly > 255: + self.error("more than 255 arguments", arguments_node) + # process args + i = 0 have_default = False while i < child_count: - argument = arguments_node.get_child(i) - arg_type = argument.type - if arg_type == syms.fpdef: - parenthesized = False - complex_args = False - while True: - if i + 1 < child_count and \ - arguments_node.get_child(i + 1).type == tokens.EQUAL: - default_node = arguments_node.get_child(i + 2) - defaults.append(self.handle_expr(default_node)) - i += 2 - have_default = True - elif have_default: - if parenthesized and not complex_args: - msg = "parenthesized arg with default" - else: - msg = ("non-default argument follows default " - "argument") - self.error(msg, arguments_node) - if argument.num_children() == 3: - sub_arg = argument.get_child(1) - if sub_arg.num_children() != 1: - complex_args = True - args.append(self.handle_arg_unpacking(sub_arg)) - else: - parenthesized = True - argument = sub_arg.get_child(0) - continue - if argument.get_child(0).type == tokens.NAME: - name_node = argument.get_child(0) - arg_name = name_node.get_value() - self.check_forbidden_name(arg_name, name_node) - name = ast.Name(arg_name, ast.Param, name_node.get_lineno(), - name_node.get_column()) - args.append(name) + arg = arguments_node.get_child(i) + arg_type = arg.type + if arg_type == syms.tfpdef or arg_type == syms.vfpdef: + if i + 1 < child_count and \ + arguments_node.get_child(i + 1).type == tokens.EQUAL: + default_node = arguments_node.get_child(i + 2) + posdefaults.append(self.handle_expr(default_node)) i += 2 - break + have_default = True + elif have_default: + msg = "non-default argument follows default argument" + self.error(msg, arguments_node) + pos.append(self.handle_arg(arg)) + i += 2 elif arg_type == tokens.STAR: + if i + 1 >= child_count: + self.error("named arguments must follow bare *", + arguments_node) name_node = arguments_node.get_child(i + 1) - variable_arg = name_node.get_value() - self.check_forbidden_name(variable_arg, name_node) - i += 3 + keywordonly_args = [] + if name_node.type == tokens.COMMA: + i += 2 + i = self.handle_keywordonly_args(arguments_node, i, kwonly, + kwdefaults) + else: + vararg = self.handle_arg(name_node) + i += 3 + if i < child_count: + next_arg_type = arguments_node.get_child(i).type + if (next_arg_type == syms.tfpdef or + next_arg_type == syms.vfpdef): + i = self.handle_keywordonly_args(arguments_node, i, + kwonly, kwdefaults) elif arg_type == tokens.DOUBLESTAR: name_node = arguments_node.get_child(i + 1) - keywords_arg = name_node.get_value() - self.check_forbidden_name(keywords_arg, name_node) + kwarg = self.handle_arg(name_node) i += 3 else: raise AssertionError("unknown node in argument list") - if not defaults: - defaults = None - if not args: - args = None - return ast.arguments(args, variable_arg, keywords_arg, defaults) + return ast.arguments(pos, vararg, kwonly, kwdefaults, kwarg, + posdefaults) - def handle_arg_unpacking(self, fplist_node): - args = [] - for i in range((fplist_node.num_children() + 1) / 2): - fpdef_node = fplist_node.get_child(i * 2) - while True: - child = fpdef_node.get_child(0) - if child.type == tokens.NAME: - arg = ast.Name(child.get_value(), ast.Store, child.get_lineno(), - child.get_column()) - args.append(arg) + def handle_keywordonly_args(self, arguments_node, i, kwonly, kwdefaults): + if kwonly is None: + self.error("named arguments must follows bare *", + arguments_node.get_child(i)) + child_count = arguments_node.num_children() + while i < child_count: + arg = arguments_node.get_child(i) + arg_type = arg.type + if arg_type == syms.vfpdef or arg_type == syms.tfpdef: + if (i + 1 < child_count and + arguments_node.get_child(i + 1).type == tokens.EQUAL): + expr = self.handle_expr(arguments_node.get_child(i + 2)) + kwdefaults.append(expr) + i += 2 else: - child = fpdef_node.get_child(1) - if child.num_children() == 1: - fpdef_node = child.get_child(0) - continue - args.append(self.handle_arg_unpacking(child)) - break - tup = ast.Tuple(args, ast.Store, fplist_node.get_lineno(), fplist_node.get_column()) - self.set_context(tup, ast.Store) - return tup + kwdefaults.append(None) + ann = None + if arg.num_children() == 3: + ann = self.handle_expr(arg.get_child(2)) + name_node = arg.get_child(0) + argname = name_node.get_value() + argname = self.new_identifier(argname) + self.check_forbidden_name(argname, name_node) + kwonly.append(ast.arg(argname, ann, arg.get_lineno(), + arg.get_column())) + i += 2 + elif arg_type == tokens.DOUBLESTAR: + return i + return i + + def handle_arg(self, arg_node): + name_node = arg_node.get_child(0) + name = self.new_identifier(name_node.get_value()) + self.check_forbidden_name(name, arg_node) + ann = None + if arg_node.num_children() == 3: + ann = self.handle_expr(arg_node.get_child(2)) + return ast.arg(name, ann, arg_node.get_lineno(), arg_node.get_column()) def handle_stmt(self, stmt): stmt_type = stmt.type @@ -632,8 +694,6 @@ stmt_type = stmt.type if stmt_type == syms.expr_stmt: return self.handle_expr_stmt(stmt) - elif stmt_type == syms.print_stmt: - return self.handle_print_stmt(stmt) elif stmt_type == syms.del_stmt: return self.handle_del_stmt(stmt) elif stmt_type == syms.pass_stmt: @@ -644,10 +704,10 @@ return self.handle_import_stmt(stmt) elif stmt_type == syms.global_stmt: return self.handle_global_stmt(stmt) + elif stmt_type == syms.nonlocal_stmt: + return self.handle_nonlocal_stmt(stmt) elif stmt_type == syms.assert_stmt: return self.handle_assert_stmt(stmt) - elif stmt_type == syms.exec_stmt: - return self.handle_exec_stmt(stmt) else: raise AssertionError("unhandled small statement") elif stmt_type == syms.compound_stmt: @@ -658,17 +718,19 @@ elif stmt_type == syms.while_stmt: return self.handle_while_stmt(stmt) elif stmt_type == syms.for_stmt: - return self.handle_for_stmt(stmt) + return self.handle_for_stmt(stmt, 0) elif stmt_type == syms.try_stmt: return self.handle_try_stmt(stmt) elif stmt_type == syms.with_stmt: - return self.handle_with_stmt(stmt) + return self.handle_with_stmt(stmt, 0) elif stmt_type == syms.funcdef: return self.handle_funcdef(stmt) elif stmt_type == syms.classdef: return self.handle_classdef(stmt) elif stmt_type == syms.decorated: return self.handle_decorated(stmt) + elif stmt_type == syms.async_stmt: + return self.handle_async_stmt(stmt) else: raise AssertionError("unhandled compound statement") else: @@ -698,12 +760,13 @@ for i in range(0, stmt.num_children() - 2, 2): target_node = stmt.get_child(i) if target_node.type == syms.yield_expr: - self.error("can't assign to yield expression", target_node) + self.error("assignment to yield expression not possible", + target_node) target_expr = self.handle_testlist(target_node) self.set_context(target_expr, ast.Store) targets.append(target_expr) value_child = stmt.get_child(-1) - if value_child.type == syms.testlist: + if value_child.type == syms.testlist_star_expr: value_expr = self.handle_testlist(value_child) else: value_expr = self.handle_expr(value_child) @@ -724,9 +787,9 @@ # Loop until we return something. while True: expr_node_type = expr_node.type - if expr_node_type == syms.test or expr_node_type == syms.old_test: + if expr_node_type == syms.test or expr_node_type == syms.test_nocond: first_child = expr_node.get_child(0) - if first_child.type in (syms.lambdef, syms.old_lambdef): + if first_child.type in (syms.lambdef, syms.lambdef_nocond): return self.handle_lambdef(first_child) elif expr_node.num_children() > 1: return self.handle_ifexp(expr_node) @@ -763,6 +826,8 @@ operands.append(self.handle_expr(expr_node.get_child(i + 1))) return ast.Compare(expr, operators, operands, expr_node.get_lineno(), expr_node.get_column()) + elif expr_node_type == syms.star_expr: + return self.handle_star_expr(expr_node) elif expr_node_type == syms.expr or \ expr_node_type == syms.xor_expr or \ expr_node_type == syms.and_expr or \ @@ -774,11 +839,19 @@ continue return self.handle_binop(expr_node) elif expr_node_type == syms.yield_expr: - if expr_node.num_children() == 2: - exp = self.handle_testlist(expr_node.get_child(1)) + is_from = False + if expr_node.num_children() > 1: + arg_node = expr_node.get_child(1) # yield arg + if arg_node.num_children() == 2: + is_from = True + expr = self.handle_expr(arg_node.get_child(1)) + else: + expr = self.handle_testlist(arg_node.get_child(0)) else: - exp = None - return ast.Yield(exp, expr_node.get_lineno(), expr_node.get_column()) + expr = None + if is_from: + return ast.YieldFrom(expr, expr_node.get_lineno(), expr_node.get_column()) + return ast.Yield(expr, expr_node.get_lineno(), expr_node.get_column()) elif expr_node_type == syms.factor: if expr_node.num_children() == 1: expr_node = expr_node.get_child(0) @@ -789,10 +862,15 @@ else: raise AssertionError("unknown expr") + def handle_star_expr(self, star_expr_node): + expr = self.handle_expr(star_expr_node.get_child(1)) + return ast.Starred(expr, ast.Load, star_expr_node.get_lineno(), + star_expr_node.get_column()) + def handle_lambdef(self, lambdef_node): expr = self.handle_expr(lambdef_node.get_child(-1)) if lambdef_node.num_children() == 3: - args = ast.arguments(None, None, None, None) + args = ast.arguments(None, None, None, None, None, None) else: args = self.handle_arguments(lambdef_node.get_child(1)) return ast.Lambda(args, expr, lambdef_node.get_lineno(), lambdef_node.get_column()) @@ -819,6 +897,11 @@ elif comp_type == tokens.GREATEREQUAL: return ast.GtE elif comp_type == tokens.NOTEQUAL: + flufl = self.compile_info.flags & consts.CO_FUTURE_BARRY_AS_BDFL + if flufl and comp_node.get_value() == '!=': + self.error('invalid comparison', comp_node) + elif not flufl and comp_node.get_value() == '<>': + self.error('invalid comparison', comp_node) return ast.NotEq elif comp_type == tokens.NAME: if comp_node.get_value() == "is": @@ -854,20 +937,6 @@ def handle_factor(self, factor_node): from pypy.interpreter.pyparser.parser import Terminal - # Fold '-' on constant numbers. - if factor_node.get_child(0).type == tokens.MINUS and \ - factor_node.num_children() == 2: - factor = factor_node.get_child(1) - if factor.type == syms.factor and factor.num_children() == 1: - power = factor.get_child(0) - if power.type == syms.power and power.num_children() == 1: - atom = power.get_child(0) - if atom.type == syms.atom and \ - atom.get_child(0).type == tokens.NUMBER: - num = atom.get_child(0) - assert isinstance(num, Terminal) - num.value = "-" + num.get_value() - return self.handle_atom(atom) expr = self.handle_expr(factor_node.get_child(1)) op_type = factor_node.get_child(0).type if op_type == tokens.PLUS: @@ -880,18 +949,35 @@ raise AssertionError("invalid factor node") return ast.UnaryOp(op, expr, factor_node.get_lineno(), factor_node.get_column()) - def handle_power(self, power_node): - atom_expr = self.handle_atom(power_node.get_child(0)) - if power_node.num_children() == 1: + def handle_atom_expr(self, atom_node): + start = 0 + num_ch = atom_node.num_children() + if atom_node.get_child(0).type == tokens.AWAIT: + start = 1 + atom_expr = self.handle_atom(atom_node.get_child(start)) + if num_ch == 1: return atom_expr - for i in range(1, power_node.num_children()): - trailer = power_node.get_child(i) + if start and num_ch == 2: + return ast.Await(atom_expr, atom_node.get_lineno(), + atom_node.get_column()) + for i in range(start+1, num_ch): + trailer = atom_node.get_child(i) if trailer.type != syms.trailer: break tmp_atom_expr = self.handle_trailer(trailer, atom_expr) tmp_atom_expr.lineno = atom_expr.lineno tmp_atom_expr.col_offset = atom_expr.col_offset atom_expr = tmp_atom_expr + if start: + return ast.Await(atom_expr, atom_node.get_lineno(), + atom_node.get_column()) + else: + return atom_expr + + def handle_power(self, power_node): + atom_expr = self.handle_atom_expr(power_node.get_child(0)) + if power_node.num_children() == 1: + return atom_expr if power_node.get_child(-1).type == syms.factor: right = self.handle_expr(power_node.get_child(-1)) atom_expr = ast.BinOp(atom_expr, ast.Pow, right, power_node.get_lineno(), @@ -900,8 +986,6 @@ def handle_slice(self, slice_node): first_child = slice_node.get_child(0) - if first_child.type == tokens.DOT: - return ast.Ellipsis() if slice_node.num_children() == 1 and first_child.type == syms.test: index = self.handle_expr(first_child) return ast.Index(index) @@ -921,10 +1005,7 @@ upper = self.handle_expr(third_child) last_child = slice_node.get_child(-1) if last_child.type == syms.sliceop: - if last_child.num_children() == 1: - step = ast.Name("None", ast.Load, last_child.get_lineno(), - last_child.get_column()) - else: + if last_child.num_children() != 1: step_child = last_child.get_child(1) if step_child.type == syms.test: step = self.handle_expr(step_child) @@ -934,12 +1015,12 @@ first_child = trailer_node.get_child(0) if first_child.type == tokens.LPAR: if trailer_node.num_children() == 2: - return ast.Call(left_expr, None, None, None, None, + return ast.Call(left_expr, None, None, trailer_node.get_lineno(), trailer_node.get_column()) else: return self.handle_call(trailer_node.get_child(1), left_expr) elif first_child.type == tokens.DOT: - attr = trailer_node.get_child(1).get_value() + attr = self.new_identifier(trailer_node.get_child(1).get_value()) return ast.Attribute(left_expr, attr, ast.Load, trailer_node.get_lineno(), trailer_node.get_column()) else: @@ -968,9 +1049,9 @@ middle.get_lineno(), middle.get_column()) def handle_call(self, args_node, callable_expr): - arg_count = 0 - keyword_count = 0 - generator_count = 0 + arg_count = 0 # position args + iterable args unpackings + keyword_count = 0 # keyword args + keyword args unpackings + generator_count = 0 for i in range(args_node.num_children()): argument = args_node.get_child(i) if argument.type == syms.argument: @@ -978,7 +1059,11 @@ arg_count += 1 elif argument.get_child(1).type == syms.comp_for: generator_count += 1 + elif argument.get_child(0).type == tokens.STAR: + arg_count += 1 else: + # argument.get_child(0).type == tokens.DOUBLESTAR + # or keyword arg keyword_count += 1 if generator_count > 1 or \ (generator_count and (keyword_count or arg_count)): @@ -989,53 +1074,66 @@ args = [] keywords = [] used_keywords = {} - variable_arg = None - keywords_arg = None + doublestars_count = 0 # just keyword argument unpackings child_count = args_node.num_children() i = 0 while i < child_count: argument = args_node.get_child(i) if argument.type == syms.argument: + expr_node = argument.get_child(0) if argument.num_children() == 1: - expr_node = argument.get_child(0) + # a positional argument if keywords: - self.error("non-keyword arg after keyword arg", + if doublestars_count: + self.error("positional argument follows " + "keyword argument unpacking", + expr_node) + else: + self.error("positional argument follows " + "keyword argument", + expr_node) + args.append(self.handle_expr(expr_node)) + elif expr_node.type == tokens.STAR: + # an iterable argument unpacking + if doublestars_count: + self.error("iterable argument unpacking follows " + "keyword argument unpacking", expr_node) - if variable_arg: - self.error("only named arguments may follow " - "*expression", expr_node) - args.append(self.handle_expr(expr_node)) + expr = self.handle_expr(argument.get_child(1)) + args.append(ast.Starred(expr, ast.Load, + expr_node.get_lineno(), + expr_node.get_column())) + elif expr_node.type == tokens.DOUBLESTAR: + # a keyword argument unpacking + i += 1 + expr = self.handle_expr(argument.get_child(1)) + keywords.append(ast.keyword(None, expr)) + doublestars_count += 1 elif argument.get_child(1).type == syms.comp_for: + # the lone generator expression args.append(self.handle_genexp(argument)) else: - keyword_node = argument.get_child(0) - keyword_expr = self.handle_expr(keyword_node) + # a keyword argument + keyword_expr = self.handle_expr(expr_node) if isinstance(keyword_expr, ast.Lambda): self.error("lambda cannot contain assignment", - keyword_node) + expr_node) elif not isinstance(keyword_expr, ast.Name): self.error("keyword can't be an expression", - keyword_node) + expr_node) keyword = keyword_expr.id if keyword in used_keywords: - self.error("keyword argument repeated", keyword_node) + self.error("keyword argument repeated", expr_node) used_keywords[keyword] = None - self.check_forbidden_name(keyword, keyword_node) + self.check_forbidden_name(keyword, expr_node) keyword_value = self.handle_expr(argument.get_child(2)) keywords.append(ast.keyword(keyword, keyword_value)) - elif argument.type == tokens.STAR: - variable_arg = self.handle_expr(args_node.get_child(i + 1)) - i += 1 - elif argument.type == tokens.DOUBLESTAR: - keywords_arg = self.handle_expr(args_node.get_child(i + 1)) - i += 1 i += 1 if not args: args = None if not keywords: keywords = None - return ast.Call(callable_expr, args, keywords, variable_arg, - keywords_arg, callable_expr.lineno, + return ast.Call(callable_expr, args, keywords, callable_expr.lineno, callable_expr.col_offset) def parse_number(self, raw): @@ -1072,10 +1170,7 @@ raw = "-" + raw w_num_str = self.space.newtext(raw) w_base = self.space.newint(base) - if raw[-1] in "lL": - tp = self.space.w_long - return self.space.call_function(tp, w_num_str, w_base) - elif raw[-1] in "jJ": + if raw[-1] in "jJ": tp = self.space.w_complex return self.space.call_function(tp, w_num_str) try: @@ -1085,43 +1180,44 @@ raise return self.space.call_function(self.space.w_float, w_num_str) + @always_inline + def handle_dictelement(self, node, i): + if node.get_child(i).type == tokens.DOUBLESTAR: + key = None + value = self.handle_expr(node.get_child(i+1)) + i += 2 + else: + key = self.handle_expr(node.get_child(i)) + value = self.handle_expr(node.get_child(i+2)) + i += 3 + return (i,key,value) + def handle_atom(self, atom_node): first_child = atom_node.get_child(0) first_child_type = first_child.type if first_child_type == tokens.NAME: - return ast.Name(first_child.get_value(), ast.Load, - first_child.get_lineno(), first_child.get_column()) + name = first_child.get_value() + if name == "None": + w_singleton = self.space.w_None + elif name == "True": + w_singleton = self.space.w_True + elif name == "False": + w_singleton = self.space.w_False + else: + name = self.new_identifier(name) + return ast.Name(name, ast.Load, first_child.get_lineno(), + first_child.get_column()) + return ast.NameConstant(w_singleton, first_child.get_lineno(), + first_child.get_column()) + # elif first_child_type == tokens.STRING: - space = self.space - encoding = self.compile_info.encoding - flags = self.compile_info.flags - unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - sub_strings_w = [] - for index in range(atom_node.num_children()): - child = atom_node.get_child(index) - try: - sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), - unicode_literals)) - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - if child is None: - child = atom_node - raise self.error('(unicode error) %s' % errmsg, child) - # This implements implicit string concatenation. - if len(sub_strings_w) > 1: - w_sub_strings = space.newlist(sub_strings_w) - w_join = space.getattr(space.newtext(""), space.newtext("join")) - final_string = space.call_function(w_join, w_sub_strings) - else: - final_string = sub_strings_w[0] - return ast.Str(final_string, atom_node.get_lineno(), atom_node.get_column()) + return fstring.string_parse_literal(self, atom_node) + # elif first_child_type == tokens.NUMBER: num_value = self.parse_number(first_child.get_value()) return ast.Num(num_value, atom_node.get_lineno(), atom_node.get_column()) + elif first_child_type == tokens.ELLIPSIS: + return ast.Ellipsis(atom_node.get_lineno(), atom_node.get_column()) elif first_child_type == tokens.LPAR: second_child = atom_node.get_child(1) if second_child.type == tokens.RPAR: @@ -1143,28 +1239,31 @@ return self.handle_listcomp(second_child) elif first_child_type == tokens.LBRACE: maker = atom_node.get_child(1) + n_maker_children = maker.num_children() if maker.type == tokens.RBRACE: + # an empty dict return ast.Dict(None, None, atom_node.get_lineno(), atom_node.get_column()) - n_maker_children = maker.num_children() - if n_maker_children == 1 or maker.get_child(1).type == tokens.COMMA: - elts = [] - for i in range(0, n_maker_children, 2): - elts.append(self.handle_expr(maker.get_child(i))) - return ast.Set(elts, atom_node.get_lineno(), atom_node.get_column()) - if maker.get_child(1).type == syms.comp_for: - return self.handle_setcomp(maker) - if (n_maker_children > 3 and - maker.get_child(3).type == syms.comp_for): - return self.handle_dictcomp(maker) - keys = [] - values = [] - for i in range(0, n_maker_children, 4): - keys.append(self.handle_expr(maker.get_child(i))) - values.append(self.handle_expr(maker.get_child(i + 2))) - return ast.Dict(keys, values, atom_node.get_lineno(), atom_node.get_column()) - elif first_child_type == tokens.BACKQUOTE: - expr = self.handle_testlist(atom_node.get_child(1)) - return ast.Repr(expr, atom_node.get_lineno(), atom_node.get_column()) + else: + is_dict = maker.get_child(0).type == tokens.DOUBLESTAR + if (n_maker_children == 1 or + (n_maker_children > 1 and + maker.get_child(1).type == tokens.COMMA)): + # a set display + return self.handle_setdisplay(maker, atom_node) + elif n_maker_children > 1 and maker.get_child(1).type == syms.comp_for: + # a set comprehension + return self.handle_setcomp(maker, atom_node) + elif (n_maker_children > (3-is_dict) and + maker.get_child(3-is_dict).type == syms.comp_for): + # a dictionary comprehension + if is_dict: + raise self.error("dict unpacking cannot be used in " + "dict comprehension", atom_node) + + return self.handle_dictcomp(maker, atom_node) + else: + # a dictionary display + return self.handle_dictdisplay(maker, atom_node) else: raise AssertionError("unknown atom") @@ -1174,7 +1273,7 @@ return self.handle_genexp(gexp_node) return self.handle_testlist(gexp_node) - def count_comp_fors(self, comp_node, for_type, if_type): + def count_comp_fors(self, comp_node): count = 0 current_for = comp_node while True: @@ -1185,10 +1284,10 @@ return count while True: first_child = current_iter.get_child(0) - if first_child.type == for_type: + if first_child.type == syms.comp_for: current_for = current_iter.get_child(0) break - elif first_child.type == if_type: + elif first_child.type == syms.comp_if: if first_child.num_children() == 3: current_iter = first_child.get_child(2) else: @@ -1196,48 +1295,40 @@ else: raise AssertionError("should not reach here") - def count_comp_ifs(self, iter_node, for_type): + def count_comp_ifs(self, iter_node): count = 0 while True: first_child = iter_node.get_child(0) - if first_child.type == for_type: + if first_child.type == syms.comp_for: return count count += 1 if first_child.num_children() == 2: return count iter_node = first_child.get_child(2) - @specialize.arg(2) - def comprehension_helper(self, comp_node, - handle_source_expr_meth="handle_expr", - for_type=syms.comp_for, if_type=syms.comp_if, - iter_type=syms.comp_iter, - comp_fix_unamed_tuple_location=False): - handle_source_expression = getattr(self, handle_source_expr_meth) - fors_count = self.count_comp_fors(comp_node, for_type, if_type) + def comprehension_helper(self, comp_node): + fors_count = self.count_comp_fors(comp_node) comps = [] for i in range(fors_count): for_node = comp_node.get_child(1) for_targets = self.handle_exprlist(for_node, ast.Store) - expr = handle_source_expression(comp_node.get_child(3)) + expr = self.handle_expr(comp_node.get_child(3)) assert isinstance(expr, ast.expr) if for_node.num_children() == 1: comp = ast.comprehension(for_targets[0], expr, None) else: - col = comp_node.get_column() - line = comp_node.get_lineno() # Modified in python2.7, see http://bugs.python.org/issue6704 - if comp_fix_unamed_tuple_location: - expr_node = for_targets[0] - assert isinstance(expr_node, ast.expr) - col = expr_node.col_offset - line = expr_node.lineno + # Fixing unamed tuple location + expr_node = for_targets[0] + assert isinstance(expr_node, ast.expr) + col = expr_node.col_offset + line = expr_node.lineno target = ast.Tuple(for_targets, ast.Store, line, col) comp = ast.comprehension(target, expr, None) if comp_node.num_children() == 5: comp_node = comp_iter = comp_node.get_child(4) - assert comp_iter.type == iter_type - ifs_count = self.count_comp_ifs(comp_iter, for_type) + assert comp_iter.type == syms.comp_iter + ifs_count = self.count_comp_ifs(comp_iter) if ifs_count: ifs = [] for j in range(ifs_count): @@ -1246,42 +1337,66 @@ if comp_if.num_children() == 3: comp_node = comp_iter = comp_if.get_child(2) comp.ifs = ifs - if comp_node.type == iter_type: + if comp_node.type == syms.comp_iter: comp_node = comp_node.get_child(0) assert isinstance(comp, ast.comprehension) comps.append(comp) return comps def handle_genexp(self, genexp_node): - elt = self.handle_expr(genexp_node.get_child(0)) - comps = self.comprehension_helper(genexp_node.get_child(1), - comp_fix_unamed_tuple_location=True) + ch = genexp_node.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) + comps = self.comprehension_helper(genexp_node.get_child(1)) return ast.GeneratorExp(elt, comps, genexp_node.get_lineno(), genexp_node.get_column()) def handle_listcomp(self, listcomp_node): - elt = self.handle_expr(listcomp_node.get_child(0)) - comps = self.comprehension_helper(listcomp_node.get_child(1), - "handle_testlist", - syms.list_for, syms.list_if, - syms.list_iter, - comp_fix_unamed_tuple_location=True) + ch = listcomp_node.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) + comps = self.comprehension_helper(listcomp_node.get_child(1)) return ast.ListComp(elt, comps, listcomp_node.get_lineno(), listcomp_node.get_column()) - def handle_setcomp(self, set_maker): - elt = self.handle_expr(set_maker.get_child(0)) - comps = self.comprehension_helper(set_maker.get_child(1), - comp_fix_unamed_tuple_location=True) - return ast.SetComp(elt, comps, set_maker.get_lineno(), set_maker.get_column()) + def handle_setcomp(self, set_maker, atom_node): + ch = set_maker.get_child(0) + elt = self.handle_expr(ch) + if isinstance(elt, ast.Starred): + self.error("iterable unpacking cannot be used in comprehension", ch) + comps = self.comprehension_helper(set_maker.get_child(1)) + return ast.SetComp(elt, comps, atom_node.get_lineno(), + atom_node.get_column()) - def handle_dictcomp(self, dict_maker): - key = self.handle_expr(dict_maker.get_child(0)) - value = self.handle_expr(dict_maker.get_child(2)) - comps = self.comprehension_helper(dict_maker.get_child(3), - comp_fix_unamed_tuple_location=True) - return ast.DictComp(key, value, comps, dict_maker.get_lineno(), - dict_maker.get_column()) + def handle_dictcomp(self, dict_maker, atom_node): + i, key, value = self.handle_dictelement(dict_maker, 0) + comps = self.comprehension_helper(dict_maker.get_child(i)) + return ast.DictComp(key, value, comps, atom_node.get_lineno(), + atom_node.get_column()) + + def handle_dictdisplay(self, node, atom_node): + keys = [] + values = [] + i = 0 + while i < node.num_children(): + i, key, value = self.handle_dictelement(node, i) + keys.append(key) + values.append(value) + i += 1 + return ast.Dict(keys, values, atom_node.get_lineno(), + atom_node.get_column()) + + def handle_setdisplay(self, node, atom_node): + elts = [] + i = 0 + while i < node.num_children(): + expr = self.handle_expr(node.get_child(i)) + elts.append(expr) + i += 2 + return ast.Set(elts, atom_node.get_lineno(), + atom_node.get_column()) def handle_exprlist(self, exprlist, context): exprs = [] From pypy.commits at gmail.com Fri Feb 9 12:18:22 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 09:18:22 -0800 (PST) Subject: [pypy-commit] pypy py3.5: no tp_compare on py3, revert 14e8d60878c2 prehaps issue #2740 still broken on py3? Message-ID: <5a7dd7de.83addf0a.1575f.6a16@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93792:ec92ed49e47f Date: 2018-02-09 12:17 -0500 http://bitbucket.org/pypy/pypy/changeset/ec92ed49e47f/ Log: no tp_compare on py3, revert 14e8d60878c2 prehaps issue #2740 still broken on py3? diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -265,11 +265,6 @@ assert cmpr.__le__(4) is NotImplemented - - def test_unhashable_when_tpcompare(self): - module = self.import_module("comparisons") - cmpr = module.OldCmpType() - raises(TypeError, hash, cmpr) def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -344,9 +344,9 @@ # two special cases where __hash__ is explicitly set to None # (which leads to an unhashable type): # 1) tp_hash == PyObject_HashNotImplemented - # 2) tp_hash == NULL and either of tp_compare or tp_richcompare are not NULL + # 2) tp_hash == NULL and tp_richcompare not NULL if hash_not_impl == func or ( - not func and (pto.c_tp_compare or pto.c_tp_richcompare)): + not func and pto.c_tp_richcompare): dict_w[method_name] = space.w_None continue else: From pypy.commits at gmail.com Fri Feb 9 16:29:58 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 13:29:58 -0800 (PST) Subject: [pypy-commit] pypy default: fix translation, cleanup Message-ID: <5a7e12d6.2696df0a.9695f.5bc8@mx.google.com> Author: Matti Picus Branch: Changeset: r93793:8b069fa659af Date: 2018-02-09 16:28 -0500 http://bitbucket.org/pypy/pypy/changeset/8b069fa659af/ Log: fix translation, cleanup diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -97,7 +97,7 @@ def q_destructor_or_custom_trace(self, typeid): if not self.get(typeid).customdata: - return lltype.nullptr(GCData.CUSTOM_DATA_STRUCT) + return lltype.nullptr(GCData.CUSTOM_FUNC_PTR.TO) return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -573,7 +573,7 @@ return compute_hash(x) for size in ([random.randrange(0, 260) for i in range(10)] + [random.randrange(260, 60000)]): - print 'PREBUILT DICTIONARY OF SIZE', size + #print 'PREBUILT DICTIONARY OF SIZE', size keys = [X() for j in range(size)] d = r_dict(keq, khash) for j in range(size): From pypy.commits at gmail.com Fri Feb 9 16:30:00 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 13:30:00 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5a7e12d8.d3b3df0a.ca3c3.3085@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93794:5c20792cdaa9 Date: 2018-02-09 16:28 -0500 http://bitbucket.org/pypy/pypy/changeset/5c20792cdaa9/ Log: merge default into py3.5 diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -97,7 +97,7 @@ def q_destructor_or_custom_trace(self, typeid): if not self.get(typeid).customdata: - return lltype.nullptr(GCData.CUSTOM_DATA_STRUCT) + return lltype.nullptr(GCData.CUSTOM_FUNC_PTR.TO) return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -573,7 +573,7 @@ return compute_hash(x) for size in ([random.randrange(0, 260) for i in range(10)] + [random.randrange(260, 60000)]): - print 'PREBUILT DICTIONARY OF SIZE', size + #print 'PREBUILT DICTIONARY OF SIZE', size keys = [X() for j in range(size)] d = r_dict(keq, khash) for j in range(size): From pypy.commits at gmail.com Fri Feb 9 17:28:57 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Feb 2018 14:28:57 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 Message-ID: <5a7e20a9.49a0df0a.42f1c.7461@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r93795:d9e0d13cf28e Date: 2018-02-09 17:28 -0500 http://bitbucket.org/pypy/pypy/changeset/d9e0d13cf28e/ Log: merge py3.5 diff too long, truncating to 2000 out of 2839 lines diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -155,9 +155,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements, uri) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements, uri) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.3 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,37 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -2,7 +2,7 @@ import __pypy__ import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,29 @@ .. branch: cpyext-datetime2 Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1422,3 +1422,11 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -207,6 +207,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -134,7 +134,7 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.3" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h --- a/pypy/module/cpyext/include/longobject.h +++ b/pypy/module/cpyext/include/longobject.h @@ -20,6 +20,9 @@ #define PyLong_AS_LONG(op) PyLong_AsLong(op) +#define _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) \ + _PyLong_AsByteArrayO((PyObject *)(v), bytes, n, little_endian, is_signed) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -3,8 +3,8 @@ cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t, CONST_STRING, ADDR, CANNOT_FAIL) from pypy.objspace.std.longobject import W_LongObject -from pypy.interpreter.error import OperationError -from rpython.rlib.rbigint import rbigint +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rbigint import rbigint, InvalidSignednessError PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long") @@ -251,3 +251,26 @@ byteorder = 'big' result = rbigint.frombytes(s, byteorder, signed != 0) return space.newlong_from_rbigint(result) + + at cpython_api([PyObject, rffi.UCHARP, rffi.SIZE_T, + rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1) +def _PyLong_AsByteArrayO(space, w_v, bytes, n, little_endian, is_signed): + n = rffi.cast(lltype.Signed, n) + little_endian = rffi.cast(lltype.Signed, little_endian) + signed = rffi.cast(lltype.Signed, is_signed) != 0 + byteorder = 'little' if little_endian else 'big' + bigint = space.bigint_w(w_v) + try: + digits = bigint.tobytes(n, byteorder, signed) + except InvalidSignednessError: # < 0 but not 'signed' + # in this case, CPython raises OverflowError even though the C + # comments say it should raise TypeError + raise oefmt(space.w_OverflowError, + "can't convert negative long to unsigned") + except OverflowError: + raise oefmt(space.w_OverflowError, + "long too big to convert") + assert len(digits) == n + for i in range(n): + bytes[i] = rffi.cast(rffi.UCHAR, digits[i]) + return 0 diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -267,7 +267,7 @@ raise oefmt(space.w_ValueError, "sequence.index(x): x not in sequence") class CPyListStrategy(ListStrategy): - erase, unerase = rerased.new_erasing_pair("empty") + erase, unerase = rerased.new_erasing_pair("cpylist") erase = staticmethod(erase) unerase = staticmethod(unerase) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1577,13 +1577,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF32String(space, unicode): - """Return a Python byte string using the UTF-32 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If @@ -1612,13 +1605,6 @@ Return NULL if an exception was raised by the codec.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF16String(space, unicode): - """Return a Python byte string using the UTF-16 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_DecodeUTF7(space, s, size, errors): """Create a Unicode object by decoding size bytes of the UTF-7 encoded string diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -259,6 +259,48 @@ assert module.from_bytearray(False, False) == 0x9ABC41 assert module.from_bytearray(False, True) == -0x6543BF + def test_asbytearray(self): + module = self.import_extension('foo', [ + ("as_bytearray", "METH_VARARGS", + """ + PyObject *result; + PyLongObject *o; + int n, little_endian, is_signed; + unsigned char *bytes; + if (!PyArg_ParseTuple(args, "O!iii", &PyLong_Type, &o, &n, + &little_endian, &is_signed)) + return NULL; + bytes = malloc(n); + if (_PyLong_AsByteArray(o, bytes, (size_t)n, + little_endian, is_signed) != 0) + { + free(bytes); + return NULL; + } + result = PyString_FromStringAndSize((const char *)bytes, n); + free(bytes); + return result; + """), + ]) + s = module.as_bytearray(0x41BC9AL, 4, True, False) + assert s == "\x9A\xBC\x41\x00" + s = module.as_bytearray(0x41BC9AL, 4, False, False) + assert s == "\x00\x41\xBC\x9A" + s = module.as_bytearray(0x41BC9AL, 3, True, False) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x41BC9AL, 3, True, True) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x9876L, 2, True, False) + assert s == "\x76\x98" + s = module.as_bytearray(0x9876L - 0x10000L, 2, True, True) + assert s == "\x76\x98" + raises(OverflowError, module.as_bytearray, + 0x9876L, 2, False, True) + raises(OverflowError, module.as_bytearray, + -1L, 2, True, False) + raises(OverflowError, module.as_bytearray, + 0x1234567L, 3, True, False) + def test_fromunicode(self): module = self.import_extension('foo', [ ("from_unicode", "METH_O", diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -218,7 +218,7 @@ if not cls.runappdirect: cls.total_mem = 0 - def add_memory_pressure(estimate): + def add_memory_pressure(estimate, object=None): assert estimate >= 0 cls.total_mem += estimate cls.orig_add_memory_pressure = [rgc.add_memory_pressure] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,13 +3,23 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr import sys import pytest class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -162,6 +172,25 @@ assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + def test_shortcut(self): + # test that instances of classes that are defined in C become an + # instance of W_BaseCPyObject and thus can be converted faster back to + # their pyobj, because they store a pointer to it directly. + if self.runappdirect: + skip("can't run with -A") + module = self.import_module(name='foo') + obj = module.fooType() + assert self._check_uses_shortcut(obj) + # W_TypeObjects use shortcut + assert self._check_uses_shortcut(object) + assert self._check_uses_shortcut(type) + # None, True, False use shortcut + assert self._check_uses_shortcut(None) + assert self._check_uses_shortcut(True) + assert self._check_uses_shortcut(False) + assert not self._check_uses_shortcut(1) + assert not self._check_uses_shortcut(object()) + def test_multiple_inheritance1(self): module = self.import_module(name='foo') obj = module.UnicodeSubtype(u'xyz') diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -359,6 +359,20 @@ m = self.import_module('_widechar') raises(ValueError, m.test_widechar) + def test_AsUTFNString(self): + module = self.import_extension('foo', [ + ("asutf8", "METH_O", "return PyUnicode_AsUTF8String(args);"), + ("asutf16", "METH_O", "return PyUnicode_AsUTF16String(args);"), + ("asutf32", "METH_O", "return PyUnicode_AsUTF32String(args);"), + ]) + u = u'sp\x09m\u1234\U00012345' + s = module.asutf8(u) + assert s == u.encode('utf-8') + s = module.asutf16(u) + assert s == u.encode('utf-16') + s = module.asutf32(u) + assert s == u.encode('utf-32') + class TestUnicode(BaseApiTest): def test_unicodeobject(self, space): @@ -448,10 +462,24 @@ lltype.free(ar, flavor='raw') def test_AsUTF8String(self, space): - w_u = space.wrap(u'sp\x09m') + w_u = space.wrap(u'sp\x09m\u1234') w_res = PyUnicode_AsUTF8String(space, w_u) assert space.type(w_res) is space.w_bytes - assert space.unwrap(w_res) == 'sp\tm' + assert space.unwrap(w_res) == 'sp\tm\xe1\x88\xb4' + + def test_AsUTF16String(self, space): + u = u'sp\x09m\u1234\U00012345' + w_u = space.wrap(u) + w_res = PyUnicode_AsUTF16String(space, w_u) + assert space.type(w_res) is space.w_bytes + assert space.unwrap(w_res) == u.encode('utf-16') + + def test_AsUTF32String(self, space): + u = u'sp\x09m\u1234\U00012345' + w_u = space.wrap(u) + w_res = PyUnicode_AsUTF32String(space, w_u) + assert space.type(w_res) is space.w_bytes + assert space.unwrap(w_res) == u.encode('utf-32') def test_decode_utf8(self, space): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -341,8 +341,12 @@ if len(slot_names) == 1: func = getattr(pto, slot_names[0]) if slot_names[0] == 'c_tp_hash': - if hash_not_impl == func: - # special case for tp_hash == PyObject_HashNotImplemented + # two special cases where __hash__ is explicitly set to None + # (which leads to an unhashable type): + # 1) tp_hash == PyObject_HashNotImplemented + # 2) tp_hash == NULL and tp_richcompare not NULL + if hash_not_impl == func or ( + not func and pto.c_tp_richcompare): dict_w[method_name] = space.w_None continue else: diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -716,7 +716,7 @@ ref[0] = rffi.cast(PyObject, py_newuni) return 0 -def make_conversion_functions(suffix, encoding): +def make_conversion_functions(suffix, encoding, only_for_asstring=False): @cpython_api([PyObject], PyObject) @func_renamer('PyUnicode_As%sString' % suffix) def PyUnicode_AsXXXString(space, w_unicode): @@ -728,6 +728,9 @@ return unicodeobject.encode_object(space, w_unicode, encoding, "strict") globals()['PyUnicode_As%sString' % suffix] = PyUnicode_AsXXXString + if only_for_asstring: + return + @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject) @func_renamer('PyUnicode_Decode%s' % suffix) def PyUnicode_DecodeXXX(space, s, size, errors): @@ -758,6 +761,8 @@ globals()['PyUnicode_Encode%s' % suffix] = PyUnicode_EncodeXXX make_conversion_functions('UTF8', 'utf-8') +make_conversion_functions('UTF16', 'utf-16', only_for_asstring=True) +make_conversion_functions('UTF32', 'utf-32', only_for_asstring=True) make_conversion_functions('ASCII', 'ascii') make_conversion_functions('Latin1', 'latin-1') if sys.platform == 'win32': diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -19,6 +19,7 @@ space.config.translation.gctransformer == "framework"): self.appleveldefs.update({ 'dump_rpy_heap': 'app_referents.dump_rpy_heap', + 'get_stats': 'app_referents.get_stats', }) self.interpleveldefs.update({ 'get_rpy_roots': 'referents.get_rpy_roots', @@ -28,6 +29,7 @@ 'get_objects': 'referents.get_objects', 'get_referents': 'referents.get_referents', 'get_referrers': 'referents.get_referrers', + '_get_stats': 'referents.get_stats', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -48,3 +48,66 @@ file.flush() fd = file.fileno() gc._dump_rpy_heap(fd) + +class GcStats(object): + def __init__(self, s): + self._s = s + for item in ('total_gc_memory', 'jit_backend_used', + 'total_memory_pressure', + 'total_allocated_memory', 'jit_backend_allocated', + 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', + 'total_rawmalloced_memory', 'nursery_size', + 'peak_arena_memory', 'peak_rawmalloced_memory'): + setattr(self, item, self._format(getattr(self._s, item))) + self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + + self._s.jit_backend_used) + self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + + self._s.jit_backend_allocated) + + def _format(self, v): + if v < 1000000: + # bit unlikely ;-) + return "%.1fkB" % (v / 1024.) + return "%.1fMB" % (v / 1024. / 1024.) + + def __repr__(self): + if self._s.total_memory_pressure != -1: + extra = "\nmemory pressure: %s" % self.total_memory_pressure + else: + extra = "" + return """Total memory consumed: + GC used: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler used: %s%s + ----------------------------- + Total: %s + + Total memory allocated: + GC allocated: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler allocated: %s%s + ----------------------------- + Total: %s + """ % (self.total_gc_memory, self.peak_memory, + self.total_arena_memory, + self.total_rawmalloced_memory, + self.nursery_size, + self.jit_backend_used, + extra, + self.memory_used_sum, + + self.total_allocated_memory, self.peak_allocated_memory, + self.peak_arena_memory, + self.peak_rawmalloced_memory, + self.nursery_size, + self.jit_backend_allocated, + extra, + self.memory_allocated_sum) + + +def get_stats(): + return GcStats(gc._get_stats()) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -1,7 +1,7 @@ -from rpython.rlib import rgc +from rpython.rlib import rgc, jit_hooks from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.error import oefmt, wrap_oserror from rpython.rlib.objectmodel import we_are_translated @@ -170,3 +170,53 @@ l = rgc.get_typeids_list() list_w = [space.newint(l[i]) for i in range(len(l))] return space.newlist(list_w) + +class W_GcStats(W_Root): + def __init__(self, memory_pressure): + if memory_pressure: + self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + else: + self.total_memory_pressure = -1 + self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY) + self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY) + self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY) + self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY) + self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None) + self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None) + self.total_arena_memory = rgc.get_stats(rgc.TOTAL_ARENA_MEMORY) + self.total_rawmalloced_memory = rgc.get_stats( + rgc.TOTAL_RAWMALLOCED_MEMORY) + self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) + self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) + self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + +W_GcStats.typedef = TypeDef("GcStats", + total_memory_pressure=interp_attrproperty("total_memory_pressure", + cls=W_GcStats, wrapfn="newint"), + total_gc_memory=interp_attrproperty("total_gc_memory", + cls=W_GcStats, wrapfn="newint"), + peak_allocated_memory=interp_attrproperty("peak_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + peak_memory=interp_attrproperty("peak_memory", + cls=W_GcStats, wrapfn="newint"), + total_allocated_memory=interp_attrproperty("total_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + jit_backend_allocated=interp_attrproperty("jit_backend_allocated", + cls=W_GcStats, wrapfn="newint"), + jit_backend_used=interp_attrproperty("jit_backend_used", + cls=W_GcStats, wrapfn="newint"), + total_arena_memory=interp_attrproperty("total_arena_memory", + cls=W_GcStats, wrapfn="newint"), + total_rawmalloced_memory=interp_attrproperty("total_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + peak_arena_memory=interp_attrproperty("peak_arena_memory", + cls=W_GcStats, wrapfn="newint"), + peak_rawmalloced_memory=interp_attrproperty("peak_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + nursery_size=interp_attrproperty("nursery_size", + cls=W_GcStats, wrapfn="newint"), +) + + at unwrap_spec(memory_pressure=bool) +def get_stats(space, memory_pressure=False): + return W_GcStats(memory_pressure) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -843,11 +843,11 @@ # Currently this is just the size of the pointer and some estimated bytes. # The struct isn't actually defined in expat.h - it is in xmlparse.c # XXX: find a good estimate of the XML_ParserStruct - rgc.add_memory_pressure(XML_Parser_SIZE + 300) if not xmlparser: raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed") parser = W_XMLParserType(space, xmlparser, w_intern) + rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser) XML_SetUnknownEncodingHandler( parser.itself, UnknownEncodingHandlerData_callback, rffi.cast(rffi.VOIDP, parser.id)) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2298,3 +2298,11 @@ else: assert lib.__loader__ is None assert lib.__spec__ is None + +def test_realize_struct_error(): + ffi = FFI() + ffi.cdef("""typedef ... foo_t; struct foo_s { void (*x)(foo_t); };""") + lib = verify(ffi, "test_realize_struct_error", """ + typedef int foo_t; struct foo_s { void (*x)(foo_t); }; + """) + py.test.raises(TypeError, ffi.new, "struct foo_s *") diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -7,7 +7,7 @@ from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, - W_DictObject, + W_DictObject, BytesDictStrategy, UnicodeDictStrategy ) from pypy.objspace.std.typeobject import MutableCell @@ -25,6 +25,10 @@ # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative +# the maximum number of attributes stored in mapdict (afterwards just use a +# dict) +LIMIT_MAP_ATTRIBUTES = 80 + class AbstractAttribute(object): _immutable_fields_ = ['terminator'] @@ -253,6 +257,9 @@ def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") + def materialize_str_dict(self, space, obj, str_dict): + raise NotImplementedError("abstract base class") + def remove_dict_entries(self, obj): raise NotImplementedError("abstract base class") @@ -272,6 +279,13 @@ def _write_terminator(self, obj, name, index, w_value): obj._get_mapdict_map().add_attr(obj, name, index, w_value) + if index == DICT and obj._get_mapdict_map().length() >= LIMIT_MAP_ATTRIBUTES: + space = self.space + w_dict = obj.getdict(space) + assert isinstance(w_dict, W_DictMultiObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, MapDictStrategy) + strategy.switch_to_text_strategy(w_dict) return True def copy(self, obj): @@ -302,6 +316,12 @@ self.devolved_dict_terminator = DevolvedDictTerminator(space, w_cls) def materialize_r_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def materialize_str_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def _make_devolved(self, space): result = Object() result.space = space result._mapdict_init_empty(self.devolved_dict_terminator) @@ -408,6 +428,14 @@ self._copy_attr(obj, new_obj) return new_obj + def materialize_str_dict(self, space, obj, str_dict): + new_obj = self.back.materialize_str_dict(space, obj, str_dict) + if self.index == DICT: + str_dict[self.name] = obj._mapdict_read_storage(self.storageindex) + else: + self._copy_attr(obj, new_obj) + return new_obj + def remove_dict_entries(self, obj): new_obj = self.back.remove_dict_entries(obj) if self.index != DICT: @@ -737,6 +765,15 @@ assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) + def switch_to_text_strategy(self, w_dict): + w_obj = self.unerase(w_dict.dstorage) + strategy = self.space.fromcache(BytesDictStrategy) + str_dict = strategy.unerase(strategy.get_empty_storage()) + w_dict.set_strategy(strategy) + w_dict.dstorage = strategy.erase(str_dict) + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None + materialize_str_dict(self.space, w_obj, str_dict) + def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) @@ -832,6 +869,11 @@ new_obj = map.materialize_r_dict(space, obj, dict_w) obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) +def materialize_str_dict(space, obj, dict_w): + map = obj._get_mapdict_map() + new_obj = map.materialize_str_dict(space, obj, dict_w) + obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) + class IteratorMixin(object): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -114,6 +114,33 @@ assert obj2.getdictvalue(space, "b") == 60 assert obj2.map is obj.map +def test_add_attribute_limit(): + for numslots in [0, 10, 100]: + cls = Class() + obj = cls.instantiate() + for i in range(numslots): + obj.setslotvalue(i, i) # some extra slots too, sometimes + # test that eventually attributes are really just stored in a dictionary + for i in range(1000): + obj.setdictvalue(space, str(i), i) + # moved to dict (which is the remaining non-slot item) + assert len(obj.storage) == 1 + numslots + + for i in range(1000): + assert obj.getdictvalue(space, str(i)) == i + for i in range(numslots): + assert obj.getslotvalue(i) == i # check extra slots + + # this doesn't happen with slots + cls = Class() + obj = cls.instantiate() + for i in range(1000): + obj.setslotvalue(i, i) + assert len(obj.storage) == 1000 + + for i in range(1000): + assert obj.getslotvalue(i) == i + def test_insert_different_orders(): cls = Class() obj = cls.instantiate() @@ -797,7 +824,6 @@ assert d == {} def test_change_class_slots(self): - skip("not supported by pypy yet") class A(object): __slots__ = ["x", "y"] @@ -815,7 +841,6 @@ assert isinstance(a, B) def test_change_class_slots_dict(self): - skip("not supported by pypy yet") class A(object): __slots__ = ["x", "__dict__"] class B(object): @@ -843,7 +868,7 @@ assert a.y == 2 d = a.__dict__ d[1] = 3 - assert d == {"x": 1, "y": 2, 1:3} + assert d == {"y": 2, 1: 3} a.__class__ = B assert a.x == 1 assert a.y == 2 diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -71,6 +71,7 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() + self.memory_pressure_types = set() self.register_builtins() diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -675,6 +675,8 @@ def _ll_1_gc_add_memory_pressure(num): llop.gc_add_memory_pressure(lltype.Void, num) + def _ll_2_gc_add_memory_pressure(num, obj): + llop.gc_add_memory_pressure(lltype.Void, num, obj) def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None): diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -83,7 +83,9 @@ has_custom_trace, fast_path_tracing, has_gcptr, - cannot_pin): + cannot_pin, + has_memory_pressure, + get_memory_pressure_ofs): self.finalizer_handlers = finalizer_handlers self.destructor_or_custom_trace = destructor_or_custom_trace self.is_old_style_finalizer = is_old_style_finalizer @@ -103,6 +105,8 @@ self.fast_path_tracing = fast_path_tracing self.has_gcptr = has_gcptr self.cannot_pin = cannot_pin + self.has_memory_pressure = has_memory_pressure + self.get_memory_pressure_ofs = get_memory_pressure_ofs def get_member_index(self, type_id): return self.member_index(type_id) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -72,6 +72,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize +from rpython.rlib import rgc from rpython.memory.gc.minimarkpage import out_of_memory # @@ -371,6 +372,7 @@ self.old_rawmalloced_objects = self.AddressStack() self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) + self.rawmalloced_peak_size = r_uint(0) self.gc_state = STATE_SCANNING # @@ -996,6 +998,8 @@ # Record the newly allocated object and its full malloced size. # The object is young or old depending on the argument. self.rawmalloced_total_size += r_uint(allocsize) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) if alloc_young: if not self.young_rawmalloced_objects: self.young_rawmalloced_objects = self.AddressDict() @@ -1023,7 +1027,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): # Decrement by 'sizehint' plus a very little bit extra. This # is needed e.g. for _rawffi, which may allocate a lot of tiny # arrays. @@ -1183,6 +1187,24 @@ """ return self.ac.total_memory_used + self.rawmalloced_total_size + def get_total_memory_alloced(self): + """ Return the total memory allocated + """ + return self.ac.total_memory_alloced + self.rawmalloced_total_size + + def get_peak_memory_alloced(self): + """ Return the peak memory ever allocated. The peaks + can be at different times, but we just don't worry for now + """ + return self.ac.peak_memory_alloced + self.rawmalloced_peak_size + + def get_peak_memory_used(self): + """ Return the peak memory GC felt ever responsible for + """ + mem_allocated = max(self.ac.peak_memory_used, + self.ac.total_memory_used) + return mem_allocated + self.rawmalloced_peak_size + def threshold_reached(self, extra=0): return (self.next_major_collection_threshold - float(self.get_total_memory_used())) < float(extra) @@ -2155,6 +2177,8 @@ # size_gc_header = self.gcheaderbuilder.size_gc_header self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize)) + self.rawmalloced_peak_size = max(self.rawmalloced_total_size, + self.rawmalloced_peak_size) self.old_rawmalloced_objects.append(arena + size_gc_header) return arena @@ -2932,6 +2956,32 @@ self.old_objects_with_weakrefs.delete() self.old_objects_with_weakrefs = new_with_weakref + def get_stats(self, stats_no): + from rpython.memory.gc import inspector + + if stats_no == rgc.TOTAL_MEMORY: + return intmask(self.get_total_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_MEMORY: + return intmask(self.get_peak_memory_used() + self.nursery_size) + elif stats_no == rgc.PEAK_ALLOCATED_MEMORY: + return intmask(self.get_peak_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY: + return intmask(self.get_total_memory_alloced() + self.nursery_size) + elif stats_no == rgc.TOTAL_MEMORY_PRESSURE: + return inspector.count_memory_pressure(self) + elif stats_no == rgc.TOTAL_ARENA_MEMORY: + return intmask(self.ac.total_memory_used) + elif stats_no == rgc.TOTAL_RAWMALLOCED_MEMORY: + return intmask(self.rawmalloced_total_size) + elif stats_no == rgc.PEAK_RAWMALLOCED_MEMORY: + return intmask(self.rawmalloced_peak_size) + elif stats_no == rgc.PEAK_ARENA_MEMORY: + return intmask(max(self.ac.peak_memory_used, + self.ac.total_memory_used)) + elif stats_no == rgc.NURSERY_SIZE: + return intmask(self.nursery_size) + return 0 + # ---------- # RawRefCount diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py --- a/rpython/memory/gc/inspector.py +++ b/rpython/memory/gc/inspector.py @@ -2,6 +2,7 @@ Utility RPython functions to inspect objects in the GC. """ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.objectmodel import free_non_gc_object from rpython.rlib import rposix, rgc, jit @@ -92,17 +93,12 @@ AddressStack = get_address_stack() -class HeapDumper(object): - _alloc_flavor_ = "raw" - BUFSIZE = 8192 # words +class BaseWalker(object): + _alloc_flavor_ = 'raw' - def __init__(self, gc, fd): + def __init__(self, gc): self.gc = gc self.gcflag = gc.gcflag_extra - self.fd = rffi.cast(rffi.INT, fd) - self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, - flavor='raw') - self.buf_count = 0 if self.gcflag == 0: self.seen = AddressDict() self.pending = AddressStack() @@ -111,8 +107,107 @@ if self.gcflag == 0: self.seen.delete() self.pending.delete() + free_non_gc_object(self) + + def add_roots(self): + self.gc.enumerate_all_roots(_hd_add_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.walk(pendingroots) + pendingroots.delete() + self.end_add_roots_marker() + + def end_add_roots_marker(self): + pass + + def add(self, obj): + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) + + def walk(self, pending): + while pending.non_empty(): + self.processobj(pending.pop()) + + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unobj(self, obj): + gc = self.gc + gc.trace(obj, self._unref, None) + + def _unref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unobj(pending.pop()) + + def finish_processing(self): + if self.gcflag != 0: + self.clear_gcflag_again() + self.unwalk(self.pending) + + def process(self): + self.add_roots() + self.walk(self.pending) + + +class MemoryPressureCounter(BaseWalker): + + def __init__(self, gc): + self.count = 0 + BaseWalker.__init__(self, gc) + + def processobj(self, obj): + gc = self.gc + typeid = gc.get_type_id(obj) + if gc.has_memory_pressure(typeid): + ofs = gc.get_memory_pressure_ofs(typeid) + val = (obj + ofs).signed[0] + self.count += val + gc.trace(obj, self._ref, None) + + def _ref(self, pointer, _): + obj = pointer.address[0] + self.add(obj) + + +class HeapDumper(BaseWalker): + BUFSIZE = 8192 # words + + def __init__(self, gc, fd): + BaseWalker.__init__(self, gc) + self.fd = rffi.cast(rffi.INT, fd) + self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE, + flavor='raw') + self.buf_count = 0 + + def delete(self): lltype.free(self.writebuffer, flavor='raw') - free_non_gc_object(self) + BaseWalker.delete(self) @jit.dont_look_inside def flush(self): @@ -143,6 +238,7 @@ self.write(0) self.write(0) self.write(-1) + end_add_roots_marker = write_marker def writeobj(self, obj): gc = self.gc @@ -152,64 +248,13 @@ self.write(gc.get_size_incl_hash(obj)) gc.trace(obj, self._writeref, None) self.write(-1) + processobj = writeobj def _writeref(self, pointer, _): obj = pointer.address[0] self.write(llmemory.cast_adr_to_int(obj)) self.add(obj) - def add(self, obj): - if self.gcflag == 0: - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) - else: - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) == 0: - hdr.tid |= self.gcflag - self.pending.append(obj) - - def add_roots(self): - self.gc.enumerate_all_roots(_hd_add_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.walk(pendingroots) - pendingroots.delete() - self.write_marker() - - def walk(self, pending): - while pending.non_empty(): - self.writeobj(pending.pop()) - - # ---------- - # A simplified copy of the above, to make sure we walk again all the - # objects to clear the 'gcflag'. - - def unwriteobj(self, obj): - gc = self.gc - gc.trace(obj, self._unwriteref, None) - - def _unwriteref(self, pointer, _): - obj = pointer.address[0] - self.unadd(obj) - - def unadd(self, obj): - assert self.gcflag != 0 - hdr = self.gc.header(obj) - if (hdr.tid & self.gcflag) != 0: - hdr.tid &= ~self.gcflag - self.pending.append(obj) - - def clear_gcflag_again(self): - self.gc.enumerate_all_roots(_hd_unadd_root, self) - pendingroots = self.pending - self.pending = AddressStack() - self.unwalk(pendingroots) - pendingroots.delete() - - def unwalk(self, pending): - while pending.non_empty(): - self.unwriteobj(pending.pop()) def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) @@ -219,15 +264,20 @@ def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) - heapdumper.add_roots() - heapdumper.walk(heapdumper.pending) + heapdumper.process() heapdumper.flush() - if heapdumper.gcflag != 0: - heapdumper.clear_gcflag_again() - heapdumper.unwalk(heapdumper.pending) + heapdumper.finish_processing() heapdumper.delete() return True +def count_memory_pressure(gc): + counter = MemoryPressureCounter(gc) + counter.process() + counter.finish_processing() + res = counter.count + counter.delete() + return res + def get_typeids_z(gc): srcaddress = gc.root_walker.gcdata.typeids_z return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR)) diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py --- a/rpython/memory/gc/minimark.py +++ b/rpython/memory/gc/minimark.py @@ -828,7 +828,7 @@ if self.max_heap_size < self.next_major_collection_threshold: self.next_major_collection_threshold = self.max_heap_size - def raw_malloc_memory_pressure(self, sizehint): + def raw_malloc_memory_pressure(self, sizehint, adr): self.next_major_collection_threshold -= sizehint if self.next_major_collection_threshold < 0: # cannot trigger a full collection now, but we can ensure diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -9,7 +9,6 @@ WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT] assert 1 << WORD_POWER_2 == WORD - # Terminology: the memory is subdivided into "arenas" containing "pages". # A page contains a number of allocated objects, called "blocks". @@ -141,6 +140,9 @@ # the total memory used, counting every block in use, without # the additional bookkeeping stuff. self.total_memory_used = r_uint(0) + self.peak_memory_used = r_uint(0) + self.total_memory_alloced = r_uint(0) + self.peak_memory_alloced = r_uint(0) def _new_page_ptr_list(self, length): @@ -294,6 +296,10 @@ # 'arena_base' points to the start of malloced memory; it might not # be a page-aligned address arena_base = llarena.arena_malloc(self.arena_size, False) + self.total_memory_alloced += self.arena_size + self.peak_memory_alloced = max(self.total_memory_alloced, + self.peak_memory_alloced) + if not arena_base: out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size @@ -321,6 +327,8 @@ """Prepare calls to mass_free_incremental(): moves the chained lists into 'self.old_xxx'. """ + self.peak_memory_used = max(self.peak_memory_used, + self.total_memory_used) self.total_memory_used = r_uint(0) # size_class = self.small_request_threshold >> WORD_POWER_2 @@ -399,6 +407,7 @@ # The whole arena is empty. Free it. llarena.arena_reset(arena.base, self.arena_size, 4) llarena.arena_free(arena.base) + self.total_memory_alloced -= self.arena_size lltype.free(arena, flavor='raw', track_allocation=False) self.arenas_count -= 1 # diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -392,23 +392,30 @@ inline = True) if getattr(GCClass, 'raw_malloc_memory_pressure', False): - def raw_malloc_memory_pressure_varsize(length, itemsize): + def raw_malloc_memory_pressure_varsize(length, itemsize, adr): totalmem = length * itemsize if totalmem > 0: - gcdata.gc.raw_malloc_memory_pressure(totalmem) + gcdata.gc.raw_malloc_memory_pressure(totalmem, adr) #else: probably an overflow -- the following rawmalloc # will fail then - def raw_malloc_memory_pressure(sizehint): - gcdata.gc.raw_malloc_memory_pressure(sizehint) + def raw_malloc_memory_pressure(sizehint, adr): + gcdata.gc.raw_malloc_memory_pressure(sizehint, adr) self.raw_malloc_memory_pressure_varsize_ptr = getfn( raw_malloc_memory_pressure_varsize, - [annmodel.SomeInteger(), annmodel.SomeInteger()], + [annmodel.SomeInteger(), annmodel.SomeInteger(), + SomeAddress()], annmodel.s_None, minimal_transform = False) self.raw_malloc_memory_pressure_ptr = getfn( raw_malloc_memory_pressure, - [annmodel.SomeInteger()], + [annmodel.SomeInteger(), SomeAddress()], annmodel.s_None, minimal_transform = False) + if getattr(GCClass, 'get_stats', False): + def get_stats(stats_no): + return gcdata.gc.get_stats(stats_no) + self.get_stats_ptr = getfn(get_stats, [annmodel.SomeInteger()], + annmodel.SomeInteger()) + self.identityhash_ptr = getfn(GCClass.identityhash.im_func, [s_gc, s_gcref], @@ -831,6 +838,39 @@ gct_fv_gc_malloc_varsize = gct_fv_gc_malloc + def gct_gc_add_memory_pressure(self, hop): + def _find_correct_type(TP): + T = TP.TO + while 'special_memory_pressure' not in T._flds: + T = T._flds['super'] + return T + + if hasattr(self, 'raw_malloc_memory_pressure_ptr'): + op = hop.spaceop + size = op.args[0] + if len(op.args) == 2: + v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure") + T = _find_correct_type(op.args[1].concretetype) + v_inst = hop.genop("cast_pointer", [op.args[1]], + resulttype=lltype.Ptr(T)) + hop.genop("bare_setfield", [v_inst, v_fld, size]) + v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]], + resulttype=llmemory.Address) + else: + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) + hop.genop("direct_call", [self.raw_malloc_memory_pressure_ptr, + size, v_adr]) + + + def gct_gc_get_stats(self, hop): + if hasattr(self, 'get_stats_ptr'): + return hop.genop("direct_call", + [self.get_stats_ptr, hop.spaceop.args[0]], + resultvar=hop.spaceop.result) + hop.genop("same_as", [rmodel.inputconst(lltype.Signed, 0)], + resultvar=hop.spaceop.result) + + def gct_gc__collect(self, hop): op = hop.spaceop if len(op.args) == 1: diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -535,12 +535,7 @@ return self.varsize_malloc_helper(hop, flags, meth, []) def gct_gc_add_memory_pressure(self, hop): - if hasattr(self, 'raw_malloc_memory_pressure_ptr'): - op = hop.spaceop - size = op.args[0] - return hop.genop("direct_call", - [self.raw_malloc_memory_pressure_ptr, - size]) + pass def varsize_malloc_helper(self, hop, flags, meth, extraargs): def intconst(c): return rmodel.inputconst(lltype.Signed, c) @@ -574,9 +569,10 @@ c_offset_to_length): if flags.get('add_memory_pressure', False): if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'): + v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL) hop.genop("direct_call", [self.raw_malloc_memory_pressure_varsize_ptr, - v_length, c_item_size]) + v_length, c_item_size, v_adr]) if c_offset_to_length is None: if flags.get('zero'): fnptr = self.raw_malloc_varsize_no_length_zero_ptr diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -1,4 +1,4 @@ -from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup, rffi from rpython.rtyper import rclass from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.debug import ll_assert @@ -21,13 +21,21 @@ # A destructor is called when the object is about to be freed. # A custom tracer (CT) enumerates the addresses that contain GCREFs. # Both are called with the address of the object as only argument. + # They're embedded in a struct that has raw_memory_offset as another + # argument, which is only valid if T_HAS_MEMORY_PRESSURE is set CUSTOM_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) CUSTOM_FUNC_PTR = lltype.Ptr(CUSTOM_FUNC) + CUSTOM_DATA_STRUCT = lltype.Struct('custom_data', + ('customfunc', CUSTOM_FUNC_PTR), + ('memory_pressure_offset', lltype.Signed), # offset to where the amount + # of owned memory pressure is stored + ) + CUSTOM_DATA_STRUCT_PTR = lltype.Ptr(CUSTOM_DATA_STRUCT) # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", ("infobits", lltype.Signed), # combination of the T_xxx consts - ("customfunc", CUSTOM_FUNC_PTR), + ("customdata", CUSTOM_DATA_STRUCT_PTR), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), hints={'immutable': True}, @@ -81,14 +89,16 @@ def q_cannot_pin(self, typeid): typeinfo = self.get(typeid) ANY = (T_HAS_GCPTR | T_IS_WEAKREF) - return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customfunc) + return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customdata) def q_finalizer_handlers(self): adr = self.finalizer_handlers # set from framework.py or gcwrapper.py return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY)) def q_destructor_or_custom_trace(self, typeid): - return self.get(typeid).customfunc + if not self.get(typeid).customdata: + return lltype.nullptr(GCData.CUSTOM_FUNC_PTR.TO) + return self.get(typeid).customdata.customfunc def q_is_old_style_finalizer(self, typeid): typeinfo = self.get(typeid) @@ -139,6 +149,15 @@ infobits = self.get(typeid).infobits return infobits & T_ANY_SLOW_FLAG == 0 + def q_has_memory_pressure(self, typeid): + infobits = self.get(typeid).infobits + return infobits & T_HAS_MEMORY_PRESSURE != 0 + + def q_get_memory_pressure_ofs(self, typeid): + infobits = self.get(typeid).infobits + assert infobits & T_HAS_MEMORY_PRESSURE != 0 + return self.get(typeid).customdata.memory_pressure_offset + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -159,7 +178,9 @@ self.q_has_custom_trace, self.q_fast_path_tracing, self.q_has_gcptr, - self.q_cannot_pin) + self.q_cannot_pin, + self.q_has_memory_pressure, + self.q_get_memory_pressure_ofs) def _has_got_custom_trace(self, typeid): type_info = self.get(typeid) @@ -176,8 +197,9 @@ T_HAS_CUSTOM_TRACE = 0x200000 T_HAS_OLDSTYLE_FINALIZER = 0x400000 T_HAS_GCPTR = 0x1000000 -T_KEY_MASK = intmask(0xFE000000) # bug detection only -T_KEY_VALUE = intmask(0x5A000000) # bug detection only +T_HAS_MEMORY_PRESSURE = 0x2000000 # first field is memory pressure field +T_KEY_MASK = intmask(0xFC000000) # bug detection only +T_KEY_VALUE = intmask(0x58000000) # bug detection only def _check_valid_type_info(p): ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id") @@ -192,6 +214,25 @@ ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") +def has_special_memory_pressure(TYPE): + if TYPE._is_varsize(): + return False + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return True + if 'super' not in T._flds: + return False + T = T._flds['super'] + +def get_memory_pressure_ofs(TYPE): + T = TYPE + while True: + if 'special_memory_pressure' in T._flds: + return llmemory.offsetof(T, 'special_memory_pressure') + if 'super' not in T._flds: + assert False, "get_ and has_memory_pressure disagree" + T = T._flds['super'] def encode_type_shape(builder, info, TYPE, index): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" @@ -202,12 +243,18 @@ infobits |= T_HAS_GCPTR # fptrs = builder.special_funcptr_for_type(TYPE) - if fptrs: + if fptrs or has_special_memory_pressure(TYPE): + customdata = lltype.malloc(GCData.CUSTOM_DATA_STRUCT, flavor='raw', + immortal=True) + info.customdata = customdata if "destructor" in fptrs: - info.customfunc = fptrs["destructor"] + customdata.customfunc = fptrs["destructor"] if "old_style_finalizer" in fptrs: - info.customfunc = fptrs["old_style_finalizer"] + customdata.customfunc = fptrs["old_style_finalizer"] infobits |= T_HAS_OLDSTYLE_FINALIZER + if has_special_memory_pressure(TYPE): + infobits |= T_HAS_MEMORY_PRESSURE + info.customdata.memory_pressure_offset = get_memory_pressure_ofs(TYPE) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -83,9 +83,9 @@ def gettypeid(self, obj): return self.get_type_id(lltype.typeOf(obj).TO) - def add_memory_pressure(self, size): + def add_memory_pressure(self, size, adr): if hasattr(self.gc, 'raw_malloc_memory_pressure'): - self.gc.raw_malloc_memory_pressure(size) + self.gc.raw_malloc_memory_pressure(size, adr) def shrink_array(self, p, smallersize): if hasattr(self.gc, 'shrink_array'): diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -695,9 +695,7 @@ return NULLRBIGINT if asize == 1: - if a._digits[0] == NULLDIGIT: - return NULLRBIGINT - elif a._digits[0] == ONEDIGIT: + if a._digits[0] == ONEDIGIT: return rbigint(b._digits[:b.size], a.sign * b.sign, b.size) elif bsize == 1: res = b.widedigit(0) * a.widedigit(0) @@ -784,35 +782,15 @@ @jit.elidable def mod(self, other): + if other.sign == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT - if other.sign != 0 and other.numdigits() == 1: - digit = other.digit(0) - if digit == 1: - return NULLRBIGINT - elif digit == 2: - modm = self.digit(0) & 1 - if modm: - return ONENEGATIVERBIGINT if other.sign == -1 else ONERBIGINT - return NULLRBIGINT - elif digit & (digit - 1) == 0: - mod = self.int_and_(digit - 1) - else: - # Perform - size = self.numdigits() - 1 - if size > 0: - rem = self.widedigit(size) - size -= 1 - while size >= 0: - rem = ((rem << SHIFT) + self.widedigit(size)) % digit - size -= 1 - else: - rem = self.digit(0) % digit - - if rem == 0: - return NULLRBIGINT - mod = rbigint([_store_digit(rem)], -1 if self.sign < 0 else 1, 1) + if other.numdigits() == 1: + otherint = other.digit(0) * other.sign + assert int_in_valid_range(otherint) + return self.int_mod(otherint) else: div, mod = _divrem(self, other) if mod.sign * other.sign == -1: @@ -821,6 +799,8 @@ @jit.elidable def int_mod(self, other): + if other == 0: + raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT @@ -828,7 +808,7 @@ # Fallback to long. return self.mod(rbigint.fromint(other)) - elif other != 0: + if 1: # preserve indentation to preserve history digit = abs(other) if digit == 1: return NULLRBIGINT From pypy.commits at gmail.com Sat Feb 10 18:59:42 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 10 Feb 2018 15:59:42 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: merge msvc14 into branch Message-ID: <5a7f876e.e3aedf0a.87772.99aa@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93796:a1fe6b1111f9 Date: 2018-02-10 18:58 -0500 http://bitbucket.org/pypy/pypy/changeset/a1fe6b1111f9/ Log: merge msvc14 into branch diff too long, truncating to 2000 out of 2750 lines diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -42,7 +42,7 @@ Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -132,243 +132,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -105,14 +105,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -31,7 +31,7 @@ from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -97,25 +97,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,9 +1,7 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... -import errno import sys if 'posix' in sys.builtin_module_names: import posix @@ -114,13 +112,6 @@ columns = structseqfield(0, "width of the terminal window in characters") lines = structseqfield(1, "height of the terminal window in characters") -if osname == 'posix': - # POSIX: we want to check the file descriptor when fdopen() is called, - # not later when we read or write data. So we call fstat(), letting - # it raise if fd is invalid. - _validate_fd = posix.fstat -else: - _validate_fd = validate_fd class times_result(metaclass=structseqtype): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -16,7 +16,8 @@ if os.name != 'nt': USEMODULES += ['fcntl', 'select', '_posixsubprocess', '_socket'] else: - USEMODULES += ['_rawffi', 'thread'] + # On windows, os.popen uses the subprocess module + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -221,9 +221,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -332,20 +333,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { return _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -359,8 +374,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -296,7 +296,8 @@ def get_libc_name(): return rwin32.GetModuleFileName(get_libc_handle()) - assert "msvcr" in get_libc_name().lower(), \ + libc_name = get_libc_name().lower() + assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \ "Suspect msvcrt library: %s" % (get_libc_name(),) elif _MINGW: def get_libc_name(): diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,12 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3.3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { intptr_t osfhnd; @@ -95,6 +96,46 @@ errno = EBADF; return 0; } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: @@ -193,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): - # CPython Issue #11395, PyPy Issue #2636: the Windows console - # returns an error (12: not enough space error) on writing into - # stdout if stdout mode is binary and the length is greater than - # 66,000 bytes (or less, depending on heap usage). Can't easily - # test that, because we need 'fd' to be non-redirected... - count = 32767 - elif count > 0x7fffffff: - count = 0x7fffffff - return count -else: - def is_valid_fd(fd): - return 1 - - @enforceargs(int) - def validate_fd(fd): - pass - - def _bound_for_write(fd, count): - return count - -def closerange(fd_low, fd_high): - # this behaves like os.closerange() from Python 2.6. - for fd in xrange(fd_low, fd_high): - try: - if is_valid_fd(fd): - os.close(fd) - except OSError: - pass - if _WIN32: includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] @@ -258,11 +255,80 @@ if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'): includes.append('sys/ttycom.h') libraries = ['util'] + eci = ExternalCompilationInfo( includes=includes, libraries=libraries, ) +def external(name, args, result, compilation_info=eci, **kwds): + return rffi.llexternal(name, args, result, + compilation_info=compilation_info, **kwds) + + +if os.name == 'nt': + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + rffi.INT, compilation_info=errno_eci, + )) + c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", + [], rffi.VOIDP, compilation_info=errno_eci)) + c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph", + [rffi.VOIDP], lltype.Void, + compilation_info=errno_eci)) + + @enforceargs(int) + def _validate_fd(fd): + if not is_valid_fd(fd): + from errno import EBADF + raise OSError(EBADF, 'Bad file descriptor') + + class FdValidator(object): + + def __init__(self, fd): + _validate_fd(fd) + + def __enter__(self): + self.invalid_param_hndlr = c_enter_suppress_iph() + return self + + def __exit__(self, *args): + c_exit_suppress_iph(self.invalid_param_hndlr) + + def _bound_for_write(fd, count): + if count > 32767 and c_isatty(fd): + # CPython Issue #11395, PyPy Issue #2636: the Windows console + # returns an error (12: not enough space error) on writing into + # stdout if stdout mode is binary and the length is greater than + # 66,000 bytes (or less, depending on heap usage). Can't easily + # test that, because we need 'fd' to be non-redirected... + count = 32767 + elif count > 0x7fffffff: + count = 0x7fffffff + return count +else: + class FdValidator(object): + + def __init__(self, fd): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def _bound_for_write(fd, count): + return count + +def closerange(fd_low, fd_high): + # this behaves like os.closerange() from Python 2.6. + for fd in xrange(fd_low, fd_high): + try: + with FdValidator(fd): + os.close(fd) + except OSError: + pass + class CConfig: _compilation_info_ = eci SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET') @@ -315,10 +381,6 @@ config = rffi_platform.configure(CConfig) globals().update(config) -def external(name, args, result, compilation_info=eci, **kwds): - return rffi.llexternal(name, args, result, - compilation_info=compilation_info, **kwds) - # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. @@ -410,12 +472,12 @@ return result def _dup(fd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup(fd) - else: - res = c_dup_noninheritable(fd) - return res + with FdValidator(fd): + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res @replace_os_function('dup') def dup(fd, inheritable=True): @@ -424,12 +486,12 @@ @replace_os_function('dup2') def dup2(fd, newfd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup2(fd, newfd) - else: - res = c_dup2_noninheritable(fd, newfd) - handle_posix_error('dup2', res) + with FdValidator(fd): + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -457,25 +519,26 @@ def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - got = handle_posix_error('read', c_read(fd, void_buf, count)) - return buf.str(got) + with FdValidator(fd): + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + got = handle_posix_error('read', c_read(fd, void_buf, count)) + return buf.str(got) @replace_os_function('write') @enforceargs(int, None) def write(fd, data): count = len(data) - validate_fd(fd) - count = _bound_for_write(fd, count) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('write', c_write(fd, buf, count)) + with FdValidator(fd): + count = _bound_for_write(fd, count) + with rffi.scoped_nonmovingbuffer(data) as buf: + ret = c_write(fd, buf, count) + return handle_posix_error('write', ret) @replace_os_function('close') def close(fd): - validate_fd(fd) - handle_posix_error('close', c_close(fd)) + with FdValidator(fd): + handle_posix_error('close', c_close(fd)) c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, @@ -483,15 +546,15 @@ @replace_os_function('lseek') def lseek(fd, pos, how): - validate_fd(fd) - if SEEK_SET is not None: - if how == 0: - how = SEEK_SET - elif how == 1: - how = SEEK_CUR - elif how == 2: - how = SEEK_END - return handle_posix_error('lseek', c_lseek(fd, pos, how)) + with FdValidator(fd): + if SEEK_SET is not None: + if how == 0: + how = SEEK_SET + elif how == 1: + how = SEEK_CUR + elif how == 2: + how = SEEK_END + return handle_posix_error('lseek', c_lseek(fd, pos, how)) if not _WIN32: c_pread = external('pread', @@ -505,7 +568,6 @@ def pread(fd, count, offset): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) @@ -513,7 +575,6 @@ @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) - validate_fd(fd) with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) @@ -524,7 +585,6 @@ @enforceargs(int, None, None) def posix_fallocate(fd, offset, length): - validate_fd(fd) return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length)) if HAVE_FADVISE: @@ -546,7 +606,6 @@ @enforceargs(int, None, None, int) def posix_fadvise(fd, offset, length, advice): - validate_fd(fd) error = c_posix_fadvise(fd, offset, length, advice) error = widen(error) if error != 0: @@ -557,7 +616,6 @@ save_err=rffi.RFFI_SAVE_ERRNO) @enforceargs(int, None, None) def lockf(fd, cmd, length): - validate_fd(fd) return handle_posix_error('lockf', c_lockf(fd, cmd, length)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, @@ -571,18 +629,18 @@ @replace_os_function('ftruncate') def ftruncate(fd, length): - validate_fd(fd) - handle_posix_error('ftruncate', c_ftruncate(fd, length)) + with FdValidator(fd): + handle_posix_error('ftruncate', c_ftruncate(fd, length)) @replace_os_function('fsync') def fsync(fd): - validate_fd(fd) - handle_posix_error('fsync', c_fsync(fd)) + with FdValidator(fd): + handle_posix_error('fsync', c_fsync(fd)) @replace_os_function('fdatasync') def fdatasync(fd): - validate_fd(fd) - handle_posix_error('fdatasync', c_fdatasync(fd)) + with FdValidator(fd): + handle_posix_error('fdatasync', c_fdatasync(fd)) def sync(): c_sync() @@ -649,8 +707,8 @@ @replace_os_function('fchdir') def fchdir(fd): - validate_fd(fd) - handle_posix_error('fchdir', c_fchdir(fd)) + with FdValidator(fd): + handle_posix_error('fchdir', c_fchdir(fd)) @replace_os_function('access') @specialize.argtype(0) @@ -1090,9 +1148,9 @@ @replace_os_function('isatty') def isatty(fd): - if not is_valid_fd(fd): - return False - return c_isatty(fd) != 0 + with FdValidator(fd): + return c_isatty(fd) != 0 + return False c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -117,7 +117,7 @@ from rpython.translator.platform import host_factory static_platform = host_factory() if static_platform.name == 'msvc': - defines += ' PROCESS_QUERY_LIMITED_INFORMATION' + defines += ' PROCESS_QUERY_LIMITED_INFORMATION' for name in defines.split(): locals()[name] = rffi_platform.ConstantInteger(name) @@ -171,7 +171,7 @@ function, if that C function was declared with the flag llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). Functions without that flag don't change the saved LastError. - Alternatively, if the function was declared + Alternatively, if the function was declared RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, then the value of the C-level WSAGetLastError() is saved instead (into the same "saved alt LastError" variable). @@ -218,9 +218,9 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) def get_osfhandle(fd): - from rpython.rlib.rposix import validate_fd - validate_fd(fd) - handle = _get_osfhandle(fd) + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + handle = _get_osfhandle(fd) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle @@ -231,45 +231,10 @@ in the dict.""" # Prior to Visual Studio 8, the MSVCRT dll doesn't export the # _dosmaperr() function, which is available only when compiled - # against the static CRT library. - from rpython.translator.platform import host_factory - static_platform = host_factory() - if static_platform.name == 'msvc': - static_platform.cflags = ['/MT'] # static CRT - static_platform.version = 0 # no manifest - cfile = udir.join('dosmaperr.c') - cfile.write(r''' - #include - #include - #include - #ifdef __GNUC__ - #define _dosmaperr mingw_dosmaperr - #endif - int main() - { - int i; - for(i=1; i < 65000; i++) { - _dosmaperr(i); - if (errno == EINVAL) { - /* CPython issue #12802 */ - if (i == ERROR_DIRECTORY) - errno = ENOTDIR; - else - continue; - } - printf("%d\t%d\n", i, errno); - } - return 0; - }''') - try: - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - except (CompilationError, WindowsError): - # Fallback for the mingw32 compiler - assert static_platform.name == 'mingw32' - errors = { + # against the static CRT library. After Visual Studio 9, this + # private function seems to be gone, so use a static map, from + # CPython PC/errmap.h + errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, @@ -279,12 +244,8 @@ 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, - 202: 8, 206: 2, 215: 11, 267: 20, 1816: 12, + 202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, } - else: - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) return errors, errno.EINVAL # A bit like strerror... @@ -299,7 +260,7 @@ buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), @@ -330,7 +291,7 @@ buf[0] = lltype.nullptr(rffi.CWCHARP.TO) try: msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -15,19 +15,6 @@ py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) -def test_get_osfhandle_raising(): - #try to test what kind of exception get_osfhandle raises w/out fd validation - py.test.skip('Crashes python') - fid = open(str(udir.join('validate_test.txt')), 'w') - fd = fid.fileno() - fid.close() - def validate_fd(fd): - return 1 - _validate_fd = rwin32.validate_fd - rwin32.validate_fd = validate_fd - raises(WindowsError, rwin32.get_osfhandle, fd) - rwin32.validate_fd = _validate_fd - def test_open_process(): pid = rwin32.GetCurrentProcessId() assert pid != 0 diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py new file mode 100644 --- /dev/null +++ b/rpython/tool/setuptools_msvc.py @@ -0,0 +1,1308 @@ +""" +Improved support for Microsoft Visual C++ compilers. + +Known supported compilers: +-------------------------- +Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + +Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + +Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) +""" + +# copied as-is from python3 +# modified to: +# - commented out monkey patching +# - lookup() returns str not unicode + +import os +import sys +import platform +import itertools +import distutils.errors +from pkg_resources.extern.packaging.version import LegacyVersion + +from setuptools.extern.six.moves import filterfalse + +#from .monkey import get_unpatched + +if platform.system() == 'Windows': + from setuptools.extern.six.moves import winreg + safe_env = os.environ +else: + """ + Mock winreg and environ so the module can be imported + on this platform. + """ + + class winreg: + HKEY_USERS = None + HKEY_CURRENT_USER = None + HKEY_LOCAL_MACHINE = None + HKEY_CLASSES_ROOT = None + + safe_env = dict() + +_msvc9_suppress_errors = ( + # msvc9compiler isn't available on some platforms + ImportError, + + # msvc9compiler raises DistutilsPlatformError in some + # environments. See #1118. + distutils.errors.DistutilsPlatformError, +) + +try: + from distutils.msvc9compiler import Reg +except _msvc9_suppress_errors: + pass + + +def msvc9_find_vcvarsall(version): + """ + Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone + compiler build for Python (VCForPython). Fall back to original behavior + when the standalone compiler is not available. + + Redirect the path of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + + Parameters + ---------- + version: float + Required Microsoft Visual C++ version. + + Return + ------ + vcvarsall.bat path: str + """ + VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) + try: + # Per-user installs register the compiler path here + productdir = Reg.get_value(key, "installdir") + except KeyError: + try: + # All-user installs on a 64-bit system register here + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") + except KeyError: + productdir = None + + if productdir: + vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + + return get_unpatched(msvc9_find_vcvarsall)(version) + + +def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): + """ + Patched "distutils.msvc9compiler.query_vcvarsall" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + + Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + + Parameters + ---------- + ver: float + Required Microsoft Visual C++ version. + arch: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environement from vcvarsall.bat (Classical way) + #try: + # orig = get_unpatched(msvc9_query_vcvarsall) + # return orig(ver, arch, *args, **kwargs) + #except distutils.errors.DistutilsPlatformError: + # # Pass error if Vcvarsall.bat is missing + # pass + #except ValueError: + # # Pass error if environment not set after executing vcvarsall.bat + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(arch, ver).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, ver, arch) + raise + + +def msvc14_get_vc_env(plat_spec): + """ + Patched "distutils._msvccompiler._get_vc_env" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) + + Parameters + ---------- + plat_spec: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environment from vcvarsall.bat (Classical way) + #try: + # return get_unpatched(msvc14_get_vc_env)(plat_spec) + #except distutils.errors.DistutilsPlatformError: + # Pass error Vcvarsall.bat is missing + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, 14.0) + raise + + +def msvc14_gen_lib_options(*args, **kwargs): + """ + Patched "distutils._msvccompiler.gen_lib_options" for fix + compatibility between "numpy.distutils" and "distutils._msvccompiler" + (for Numpy < 1.11.2) + """ + if "numpy.distutils" in sys.modules: + import numpy as np + if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'): + return np.distutils.ccompiler.gen_lib_options(*args, **kwargs) + return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs) + + +def _augment_exception(exc, version, arch=''): + """ + Add details to the exception message to help guide the user + as to what action will resolve it. + """ + # Error if MSVC++ directory not found or environment not set + message = exc.args[0] + + if "vcvarsall" in message.lower() or "visual c" in message.lower(): + # Special error message if MSVC++ not installed + tmpl = 'Microsoft Visual C++ {version:0.1f} is required.' + message = tmpl.format(**locals()) + msdownload = 'www.microsoft.com/download/details.aspx?id=%d' + if version == 9.0: + if arch.lower().find('ia64') > -1: + # For VC++ 9.0, if IA64 support is needed, redirect user + # to Windows SDK 7.0 + message += ' Get it with "Microsoft Windows SDK 7.0": ' + message += msdownload % 3138 + else: + # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : + # This redirection link is maintained by Microsoft. + # Contact vspython at microsoft.com if it needs updating. + message += ' Get it from http://aka.ms/vcpython27' + elif version == 10.0: + # For VC++ 10.0 Redirect user to Windows SDK 7.1 + message += ' Get it with "Microsoft Windows SDK 7.1": ' + message += msdownload % 8279 + elif version >= 14.0: + # For VC++ 14.0 Redirect user to Visual C++ Build Tools + message += (' Get it with "Microsoft Visual C++ Build Tools": ' + r'http://landinghub.visualstudio.com/' + 'visual-cpp-build-tools') + + exc.args = (message, ) + + +class PlatformInfo: + """ + Current and Target Architectures informations. + + Parameters + ---------- + arch: str + Target architecture. + """ + current_cpu = safe_env.get('processor_architecture', '').lower() + + def __init__(self, arch): + self.arch = arch.lower().replace('x64', 'amd64') + + @property + def target_cpu(self): + return self.arch[self.arch.find('_') + 1:] + + def target_is_x86(self): + return self.target_cpu == 'x86' + + def current_is_x86(self): + return self.current_cpu == 'x86' + + def current_dir(self, hidex86=False, x64=False): + """ + Current platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\target', or '' (see hidex86 parameter) + """ + return ( + '' if (self.current_cpu == 'x86' and hidex86) else + r'\x64' if (self.current_cpu == 'amd64' and x64) else + r'\%s' % self.current_cpu + ) + + def target_dir(self, hidex86=False, x64=False): + r""" + Target platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\current', or '' (see hidex86 parameter) + """ + return ( + '' if (self.target_cpu == 'x86' and hidex86) else + r'\x64' if (self.target_cpu == 'amd64' and x64) else + r'\%s' % self.target_cpu + ) + + def cross_dir(self, forcex86=False): + r""" + Cross platform specific subfolder. + + Parameters + ---------- + forcex86: bool + Use 'x86' as current architecture even if current acritecture is + not x86. + + Return + ------ + subfolder: str + '' if target architecture is current architecture, + '\current_target' if not. + """ + current = 'x86' if forcex86 else self.current_cpu + return ( + '' if self.target_cpu == current else + self.target_dir().replace('\\', '\\%s_' % current) + ) + + +class RegistryInfo: + """ + Microsoft Visual Studio related registry informations. + + Parameters + ---------- + platform_info: PlatformInfo + "PlatformInfo" instance. + """ + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + + def __init__(self, platform_info): + self.pi = platform_info + + @property + def visualstudio(self): + """ + Microsoft Visual Studio root registry key. + """ + return 'VisualStudio' + + @property + def sxs(self): + """ + Microsoft Visual Studio SxS registry key. + """ + return os.path.join(self.visualstudio, 'SxS') + + @property + def vc(self): + """ + Microsoft Visual C++ VC7 registry key. + """ + return os.path.join(self.sxs, 'VC7') + + @property + def vs(self): + """ + Microsoft Visual Studio VS7 registry key. + """ + return os.path.join(self.sxs, 'VS7') + + @property + def vc_for_python(self): + """ + Microsoft Visual C++ for Python registry key. + """ + return r'DevDiv\VCForPython' + + @property + def microsoft_sdk(self): + """ + Microsoft SDK registry key. + """ + return 'Microsoft SDKs' + + @property + def windows_sdk(self): + """ + Microsoft Windows/Platform SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'Windows') + + @property + def netfx_sdk(self): + """ + Microsoft .NET Framework SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'NETFXSDK') + + @property + def windows_kits_roots(self): + """ + Microsoft Windows Kits Roots registry key. + """ + return r'Windows Kits\Installed Roots' + + def microsoft(self, key, x86=False): + """ + Return key in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + x86: str + Force x86 software registry. + + Return + ------ + str: value + """ + node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node' + return os.path.join('Software', node64, 'Microsoft', key) + + def lookup(self, key, name): + """ + Look for values in registry in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + name: str + Value name to find. + + Return + ------ + str: value + """ + KEY_READ = winreg.KEY_READ + openkey = winreg.OpenKey + ms = self.microsoft + for hkey in self.HKEYS: + try: + bkey = openkey(hkey, ms(key), 0, KEY_READ) + except (OSError, IOError): + if not self.pi.current_is_x86(): + try: + bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + except (OSError, IOError): + continue + else: + continue + try: + # modified to return str for python2 + return winreg.QueryValueEx(bkey, name)[0].encode('utf8') + except (OSError, IOError): + pass + + +class SystemInfo: + """ + Microsoft Windows and Visual Studio related system inormations. + + Parameters + ---------- + registry_info: RegistryInfo + "RegistryInfo" instance. + vc_ver: float + Required Microsoft Visual C++ version. + """ + + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + WinDir = safe_env.get('WinDir', '') + ProgramFiles = safe_env.get('ProgramFiles', '') + ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + + def __init__(self, registry_info, vc_ver=None): + self.ri = registry_info + self.pi = self.ri.pi + self.vc_ver = vc_ver or self._find_latest_available_vc_ver() + + def _find_latest_available_vc_ver(self): + try: + return self.find_available_vc_vers()[-1] + except IndexError: + err = 'No Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + def find_available_vc_vers(self): + """ + Find all available Microsoft Visual C++ versions. + """ + ms = self.ri.microsoft + vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) + vc_vers = [] + for hkey in self.ri.HKEYS: + for key in vckeys: + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + try: + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + for i in range(subkeys): + try: + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + return sorted(vc_vers) + + @property + def VSInstallDir(self): + """ + Microsoft Visual Studio directory. + """ + # Default path + name = 'Microsoft Visual Studio %0.1f' % self.vc_ver + default = os.path.join(self.ProgramFilesx86, name) + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + + @property + def VCInstallDir(self): + """ + Microsoft Visual C++ directory. + """ + self.VSInstallDir + + guess_vc = self._guess_vc() or self._guess_vc_legacy() + + # Try to get "VC++ for Python" path from registry as default path + reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc + + # Try to get path from registry, if fail use default path + path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + + if not os.path.isdir(path): + msg = 'Microsoft Visual C++ directory not found' + raise distutils.errors.DistutilsPlatformError(msg) + + return path + + def _guess_vc(self): + """ + Locate Visual C for 2017 + """ + if self.vc_ver <= 14.0: + return + + default = r'VC\Tools\MSVC' + guess_vc = os.path.join(self.VSInstallDir, default) + # Subdir with VC exact version as name + try: + vc_exact_ver = os.listdir(guess_vc)[-1] + return os.path.join(guess_vc, vc_exact_ver) + except (OSError, IOError, IndexError): + pass + + def _guess_vc_legacy(self): + """ + Locate Visual C for versions prior to 2017 + """ + default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver + return os.path.join(self.ProgramFilesx86, default) + + @property + def WindowsSdkVersion(self): + """ + Microsoft Windows SDK versions for specified MSVC++ version. + """ + if self.vc_ver <= 9.0: + return ('7.0', '6.1', '6.0a') + elif self.vc_ver == 10.0: + return ('7.1', '7.0a') + elif self.vc_ver == 11.0: + return ('8.0', '8.0a') + elif self.vc_ver == 12.0: + return ('8.1', '8.1a') + elif self.vc_ver >= 14.0: + return ('10.0', '8.1') + + @property + def WindowsSdkLastVersion(self): + """ + Microsoft Windows SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.WindowsSdkDir, 'lib')) + + @property + def WindowsSdkDir(self): + """ + Microsoft Windows SDK directory. + """ + sdkdir = '' + for ver in self.WindowsSdkVersion: + # Try to get it from registry + loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + sdkdir = self.ri.lookup(loc, 'installationfolder') + if sdkdir: + break + if not sdkdir or not os.path.isdir(sdkdir): + # Try to get "VC++ for Python" version from registry + path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + install_base = self.ri.lookup(path, 'installdir') + if install_base: + sdkdir = os.path.join(install_base, 'WinSDK') + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default new path + for ver in self.WindowsSdkVersion: + intver = ver[:ver.rfind('.')] + path = r'Microsoft SDKs\Windows Kits\%s' % (intver) + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default old path + for ver in self.WindowsSdkVersion: + path = r'Microsoft SDKs\Windows\v%s' % ver + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir: + # If fail, use Platform SDK + sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + return sdkdir + + @property + def WindowsSDKExecutablePath(self): + """ + Microsoft Windows SDK executable directory. + """ + # Find WinSDK NetFx Tools registry dir name + if self.vc_ver <= 11.0: + netfxver = 35 + arch = '' + else: + netfxver = 40 + hidex86 = True if self.vc_ver <= 12.0 else False + arch = self.pi.current_dir(x64=True, hidex86=hidex86) + fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) + + # liste all possibles registry paths + regpaths = [] + if self.vc_ver >= 14.0: + for ver in self.NetFxSdkVersion: + regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + + for ver in self.WindowsSdkVersion: + regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + + # Return installation folder from the more recent path + for path in regpaths: + execpath = self.ri.lookup(path, 'installationfolder') + if execpath: + break + return execpath + + @property + def FSharpInstallDir(self): + """ + Microsoft Visual F# directory. + """ + path = r'%0.1f\Setup\F#' % self.vc_ver + path = os.path.join(self.ri.visualstudio, path) + return self.ri.lookup(path, 'productdir') or '' + + @property + def UniversalCRTSdkDir(self): + """ + Microsoft Universal CRT SDK directory. + """ + # Set Kit Roots versions for specified MSVC++ version + if self.vc_ver >= 14.0: + vers = ('10', '81') + else: + vers = () + + # Find path of the more recent Kit + for ver in vers: + sdkdir = self.ri.lookup(self.ri.windows_kits_roots, + 'kitsroot%s' % ver) + if sdkdir: + break + return sdkdir or '' + + @property + def UniversalCRTSdkLastVersion(self): + """ + Microsoft Universal C Runtime SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.UniversalCRTSdkDir, 'lib')) + + @property + def NetFxSdkVersion(self): + """ + Microsoft .NET Framework SDK versions. + """ + # Set FxSdk versions for specified MSVC++ version + if self.vc_ver >= 14.0: + return ('4.6.1', '4.6') + else: + return () + + @property + def NetFxSdkDir(self): + """ + Microsoft .NET Framework SDK directory. + """ + for ver in self.NetFxSdkVersion: + loc = os.path.join(self.ri.netfx_sdk, ver) + sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder') + if sdkdir: + break + return sdkdir or '' + + @property + def FrameworkDir32(self): + """ + Microsoft .NET Framework 32bit directory. + """ + # Default path + guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework') + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw + + @property + def FrameworkDir64(self): + """ + Microsoft .NET Framework 64bit directory. + """ + # Default path + guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework64') + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw + + @property + def FrameworkVersion32(self): + """ + Microsoft .NET Framework 32bit versions. + """ + return self._find_dot_net_versions(32) + + @property + def FrameworkVersion64(self): + """ + Microsoft .NET Framework 64bit versions. + """ + return self._find_dot_net_versions(64) + + def _find_dot_net_versions(self, bits): + """ + Find Microsoft .NET Framework versions. + + Parameters + ---------- + bits: int + Platform number of bits: 32 or 64. + """ + # Find actual .NET version in registry + reg_ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits) + dot_net_dir = getattr(self, 'FrameworkDir%d' % bits) + ver = reg_ver or self._use_last_dir_name(dot_net_dir, 'v') or '' + + # Set .NET versions for specified MSVC++ version + if self.vc_ver >= 12.0: + frameworkver = (ver, 'v4.0') + elif self.vc_ver >= 10.0: + frameworkver = ('v4.0.30319' if ver.lower()[:2] != 'v4' else ver, + 'v3.5') + elif self.vc_ver == 9.0: + frameworkver = ('v3.5', 'v2.0.50727') + if self.vc_ver == 8.0: + frameworkver = ('v3.0', 'v2.0.50727') + return frameworkver + + def _use_last_dir_name(self, path, prefix=''): + """ + Return name of the last dir in path or '' if no dir found. + + Parameters + ---------- + path: str + Use dirs in this path + prefix: str + Use only dirs startings by this prefix + """ + matching_dirs = ( + dir_name + for dir_name in reversed(os.listdir(path)) + if os.path.isdir(os.path.join(path, dir_name)) and + dir_name.startswith(prefix) + ) + return next(matching_dirs, None) or '' + + +class EnvironmentInfo: + """ From pypy.commits at gmail.com Sat Feb 10 19:01:53 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 10 Feb 2018 16:01:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: prefer msvc14 Message-ID: <5a7f87f1.6d86df0a.2041c.f7e2@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93797:c16af036eb3c Date: 2018-02-10 19:01 -0500 http://bitbucket.org/pypy/pypy/changeset/c16af036eb3c/ Log: prefer msvc14 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -29,8 +29,9 @@ return None def Windows(cc=None, ver0=None): - if ver0 is None: - ver0 = _get_vcver0() + # disallow getting currently used compiler + #if ver0 is None: + # ver0 = _get_vcver0() return _get_compiler_type(cc, False, ver0=ver0) def Windows_x64(cc=None, ver0=None): From pypy.commits at gmail.com Sun Feb 11 04:52:29 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 11 Feb 2018 01:52:29 -0800 (PST) Subject: [pypy-commit] cffi default: More comments for e59662b013b4 Message-ID: <5a80125d.8fb1df0a.873ce.7735@mx.google.com> Author: Armin Rigo Branch: Changeset: r3095:2d01bb9db3de Date: 2018-02-11 10:52 +0100 http://bitbucket.org/cffi/cffi/changeset/2d01bb9db3de/ Log: More comments for e59662b013b4 diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -169,6 +169,13 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. if dist.py_modules is None: dist.py_modules = [] dist.py_modules.append(module_name) From pypy.commits at gmail.com Sun Feb 11 11:19:07 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 08:19:07 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix test for python3 Message-ID: <5a806cfb.2381df0a.d1ca4.5f26@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93798:8c123f06687c Date: 2018-02-11 09:00 -0500 http://bitbucket.org/pypy/pypy/changeset/8c123f06687c/ Log: fix test for python3 diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -282,24 +282,24 @@ return result; """), ]) - s = module.as_bytearray(0x41BC9AL, 4, True, False) + s = module.as_bytearray(0x41BC9A, 4, True, False) assert s == "\x9A\xBC\x41\x00" - s = module.as_bytearray(0x41BC9AL, 4, False, False) + s = module.as_bytearray(0x41BC9A, 4, False, False) assert s == "\x00\x41\xBC\x9A" - s = module.as_bytearray(0x41BC9AL, 3, True, False) + s = module.as_bytearray(0x41BC9A, 3, True, False) assert s == "\x9A\xBC\x41" - s = module.as_bytearray(0x41BC9AL, 3, True, True) + s = module.as_bytearray(0x41BC9A, 3, True, True) assert s == "\x9A\xBC\x41" - s = module.as_bytearray(0x9876L, 2, True, False) + s = module.as_bytearray(0x9876, 2, True, False) assert s == "\x76\x98" - s = module.as_bytearray(0x9876L - 0x10000L, 2, True, True) + s = module.as_bytearray(0x9876 - 0x10000, 2, True, True) assert s == "\x76\x98" raises(OverflowError, module.as_bytearray, - 0x9876L, 2, False, True) + 0x9876, 2, False, True) raises(OverflowError, module.as_bytearray, - -1L, 2, True, False) + -1, 2, True, False) raises(OverflowError, module.as_bytearray, - 0x1234567L, 3, True, False) + 0x1234567, 3, True, False) def test_fromunicode(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Sun Feb 11 11:19:09 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 08:19:09 -0800 (PST) Subject: [pypy-commit] pypy py3.5: test, fix for unicode keys. Shouldn't there be a case for ByteDictStrategy? Message-ID: <5a806cfd.50a6df0a.27887.9baf@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93799:d46f72070fa8 Date: 2018-02-11 11:18 -0500 http://bitbucket.org/pypy/pypy/changeset/d46f72070fa8/ Log: test, fix for unicode keys. Shouldn't there be a case for ByteDictStrategy? diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -767,7 +767,7 @@ def switch_to_text_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) - strategy = self.space.fromcache(BytesDictStrategy) + strategy = self.space.fromcache(UnicodeDictStrategy) str_dict = strategy.unerase(strategy.get_empty_storage()) w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(str_dict) diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -802,6 +802,18 @@ d[1] = 3 a.__dict__ = {} + def test_dict_limit(self): + class A(object): + pass + a = A() + for n in (40, 1000): + for i in range(n): + v = str(i).encode().decode('utf8') + setattr(a, v, i) + for i in range(n): + v = str(i).encode().decode('utf8') + assert (n, getattr(a, v)) == (n, i) + def test_dict_clear_bug(self): class A(object): pass From pypy.commits at gmail.com Sun Feb 11 13:31:55 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 10:31:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5: back out changeset: d46f72070fa8, solution is more complicated tahn a one-line fix Message-ID: <5a808c1b.43441c0a.4ac47.379f@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93800:71e87bfd8acd Date: 2018-02-11 13:31 -0500 http://bitbucket.org/pypy/pypy/changeset/71e87bfd8acd/ Log: back out changeset: d46f72070fa8, solution is more complicated tahn a one-line fix diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -767,7 +767,7 @@ def switch_to_text_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) - strategy = self.space.fromcache(UnicodeDictStrategy) + strategy = self.space.fromcache(BytesDictStrategy) str_dict = strategy.unerase(strategy.get_empty_storage()) w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(str_dict) diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -802,18 +802,6 @@ d[1] = 3 a.__dict__ = {} - def test_dict_limit(self): - class A(object): - pass - a = A() - for n in (40, 1000): - for i in range(n): - v = str(i).encode().decode('utf8') - setattr(a, v, i) - for i in range(n): - v = str(i).encode().decode('utf8') - assert (n, getattr(a, v)) == (n, i) - def test_dict_clear_bug(self): class A(object): pass From pypy.commits at gmail.com Sun Feb 11 13:58:04 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 10:58:04 -0800 (PST) Subject: [pypy-commit] pypy msvc14: document how to install vc14 Message-ID: <5a80923c.01c0df0a.35f1a.7d0a@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93801:87ef885dfbcf Date: 2018-02-11 13:52 -0500 http://bitbucket.org/pypy/pypy/changeset/87ef885dfbcf/ Log: document how to install vc14 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,6 +39,16 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads + Translating PyPy with Visual Studio ----------------------------------- @@ -82,8 +92,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C From pypy.commits at gmail.com Sun Feb 11 14:38:17 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 11 Feb 2018 11:38:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Issue #2747 Message-ID: <5a809ba9.28b9df0a.5a5ff.e9c0@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r93802:b256fef073df Date: 2018-02-11 20:37 +0100 http://bitbucket.org/pypy/pypy/changeset/b256fef073df/ Log: Issue #2747 unixccompiler.py thinks the compiler is not GCC because it is ``cc`` with PyPy. diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py --- a/lib-python/3/distutils/unixccompiler.py +++ b/lib-python/3/distutils/unixccompiler.py @@ -222,6 +222,10 @@ return "-L" + dir def _is_gcc(self, compiler_name): + if "__pypy__" in sys.builtin_module_names: # issue #2747 + if (compiler_name.startswith('cc') or + compiler_name.startswith('c++')): + return True return "gcc" in compiler_name or "g++" in compiler_name def runtime_library_dir_option(self, dir): From pypy.commits at gmail.com Sun Feb 11 14:52:39 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 11 Feb 2018 11:52:39 -0800 (PST) Subject: [pypy-commit] pypy default: Issue #2750 Message-ID: <5a809f07.1dbf1c0a.62973.121f@mx.google.com> Author: Armin Rigo Branch: Changeset: r93803:7eaba44b228a Date: 2018-02-11 20:52 +0100 http://bitbucket.org/pypy/pypy/changeset/7eaba44b228a/ Log: Issue #2750 Speed up array.__setslice__() for arguments that are changing the size of the array, but in such a way that it's just an array.extend(). diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1090,12 +1090,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -267,6 +267,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 From pypy.commits at gmail.com Sun Feb 11 15:39:17 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 12:39:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: temporary fix Message-ID: <5a80a9f5.42badf0a.b2257.468b@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93804:73862e53862b Date: 2018-02-11 13:34 -0500 http://bitbucket.org/pypy/pypy/changeset/73862e53862b/ Log: temporary fix diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -27,7 +27,7 @@ # the maximum number of attributes stored in mapdict (afterwards just use a # dict) -LIMIT_MAP_ATTRIBUTES = 80 +LIMIT_MAP_ATTRIBUTES = 400 class AbstractAttribute(object): From pypy.commits at gmail.com Sun Feb 11 15:39:19 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 12:39:19 -0800 (PST) Subject: [pypy-commit] pypy msvc14: document an additional download necessary for msvc14 Message-ID: <5a80a9f7.984a1c0a.13fb1.8841@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93805:fa9dd53cf815 Date: 2018-02-11 15:38 -0500 http://bitbucket.org/pypy/pypy/changeset/fa9dd53cf815/ Log: document an additional download necessary for msvc14 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -46,8 +46,12 @@ 14.0. A compact version of the compiler suite can be obtained from Microsoft_ downloads, search the page for "Build Tools for Visual Studio 2017". +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + .. _Wiki: https://wiki.python.org/moin/WindowsCompilers .. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk Translating PyPy with Visual Studio ----------------------------------- From pypy.commits at gmail.com Sun Feb 11 17:36:30 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 14:36:30 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: debugging build slave Message-ID: <5a80c56e.6294df0a.6ce96.2d2d@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93806:a7d51637ca64 Date: 2018-02-11 17:35 -0500 http://bitbucket.org/pypy/pypy/changeset/a7d51637ca64/ Log: debugging build slave diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -278,7 +278,11 @@ mfid = 2 out_arg = '-outputresource:%s;%s' % (exe_name, mfid) args = ['-nologo', '-manifest', str(temp_manifest), out_arg] - self._execute_c_compiler('mt.exe', args, exe_name) + try: + self._execute_c_compiler('mt.exe', args, exe_name) + except EnvironmentError: + print 'PATH is\n ', '\n '.join(self.c_environ['PATH'].split(';')) + raise return exe_name From pypy.commits at gmail.com Sun Feb 11 22:52:03 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Feb 2018 19:52:03 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: ignore VS140COMNTOOLS environment variable in order to set up the SDK toolkit Message-ID: <5a810f63.c35c1c0a.69b31.10ec@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93807:5a84b83fffbe Date: 2018-02-11 22:49 -0500 http://bitbucket.org/pypy/pypy/changeset/5a84b83fffbe/ Log: ignore VS140COMNTOOLS environment variable in order to set up the SDK toolkit diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -56,7 +56,10 @@ vcdict = None toolsdir = None try: - toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] + if vsver < 140: + toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] + else: + raise KeyError('always use registry values') except KeyError: # use setuptools from python3 to find tools try: From pypy.commits at gmail.com Mon Feb 12 13:16:25 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 10:16:25 -0800 (PST) Subject: [pypy-commit] pypy default: fix unicode wide/narrow test issues Message-ID: <5a81d9f9.131f1c0a.791dd.f539@mx.google.com> Author: Matti Picus Branch: Changeset: r93808:f2ff87a34f1f Date: 2018-02-12 13:14 -0500 http://bitbucket.org/pypy/pypy/changeset/f2ff87a34f1f/ Log: fix unicode wide/narrow test issues diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,5 +1,6 @@ import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be) @@ -26,7 +27,10 @@ got = decode_utf8(space, "\xed\xa0\x80\xed\xb0\x80") assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] @pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) def test_utf32_surrogates(unich): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -538,11 +538,16 @@ assert '\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 def test_badhandler(self): import codecs From pypy.commits at gmail.com Mon Feb 12 14:27:01 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:01 -0800 (PST) Subject: [pypy-commit] pypy msvc14: ignore VS140COMNTOOLS environment variable in order to set up the SDK toolkit Message-ID: <5a81ea85.c68f1c0a.3934a.6140@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93809:c62a256a0910 Date: 2018-02-11 22:49 -0500 http://bitbucket.org/pypy/pypy/changeset/c62a256a0910/ Log: ignore VS140COMNTOOLS environment variable in order to set up the SDK toolkit (grafted from 5a84b83fffbe222d785d642a521b4f99f4173965) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -55,7 +55,10 @@ vcdict = None toolsdir = None try: - toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] + if vsver < 140: + toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] + else: + raise KeyError('always use registry values') except KeyError: # use setuptools from python3 to find tools try: From pypy.commits at gmail.com Mon Feb 12 14:27:03 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:03 -0800 (PST) Subject: [pypy-commit] pypy msvc14: close branch to be merged Message-ID: <5a81ea87.0ee51c0a.df092.db6d@mx.google.com> Author: Matti Picus Branch: msvc14 Changeset: r93810:646310c36c6e Date: 2018-02-12 14:05 -0500 http://bitbucket.org/pypy/pypy/changeset/646310c36c6e/ Log: close branch to be merged From pypy.commits at gmail.com Mon Feb 12 14:27:07 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:07 -0800 (PST) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5a81ea8b.0f821c0a.ad16.d671@mx.google.com> Author: Matti Picus Branch: Changeset: r93812:449e2e93c275 Date: 2018-02-12 14:13 -0500 http://bitbucket.org/pypy/pypy/changeset/449e2e93c275/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -40,3 +40,7 @@ .. branch: memory-accounting Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows From pypy.commits at gmail.com Mon Feb 12 14:27:09 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:09 -0800 (PST) Subject: [pypy-commit] pypy py3.5-msvc14: close temporary branch Message-ID: <5a81ea8d.4ed11c0a.6443e.ed56@mx.google.com> Author: Matti Picus Branch: py3.5-msvc14 Changeset: r93813:d10a6cb00365 Date: 2018-02-12 14:16 -0500 http://bitbucket.org/pypy/pypy/changeset/d10a6cb00365/ Log: close temporary branch From pypy.commits at gmail.com Mon Feb 12 14:27:06 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:06 -0800 (PST) Subject: [pypy-commit] pypy default: merge msvc14 which allows using Visual Studio 17 compiler suite Message-ID: <5a81ea8a.28b9df0a.5a5ff.161f@mx.google.com> Author: Matti Picus Branch: Changeset: r93811:496d05d4758e Date: 2018-02-12 14:06 -0500 http://bitbucket.org/pypy/pypy/changeset/496d05d4758e/ Log: merge msvc14 which allows using Visual Studio 17 compiler suite diff too long, truncating to 2000 out of 2842 lines diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'get_hidden_tb' : 'interp_magic.get_hidden_tb', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -107,14 +107,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - def get_console_cp(space): from rpython.rlib import rwin32 # Windows only return space.newtuple([ diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -96,25 +96,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,7 +1,6 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import sys @@ -80,25 +79,27 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen + if osname == 'posix': # POSIX: we want to check the file descriptor when fdopen() is called, # not later when we read or write data. So we call fstat(), letting # it raise if fd is invalid. - _validate_fd = posix.fstat + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object + + Return an open file object connected to a file descriptor.""" + posix.fstat(fd) + return _fdopen(fd, mode, buffering) + else: - _validate_fd = validate_fd + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object -# Capture file.fdopen at import time, as some code replaces -# __builtins__.file with a custom function. -_fdopen = file.fdopen - -def fdopen(fd, mode='r', buffering=-1): - """fdopen(fd [, mode='r' [, buffering]]) -> file_object - - Return an open file object connected to a file descriptor.""" - _validate_fd(fd) - return _fdopen(fd, mode, buffering) - + Return an open file object connected to a file descriptor.""" + return _fdopen(fd, mode, buffering) def tmpfile(): """Create a temporary file. diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -18,7 +18,7 @@ USEMODULES += ['fcntl'] else: # On windows, os.popen uses the subprocess module - USEMODULES += ['_rawffi', 'thread', 'signal'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) @@ -300,8 +300,13 @@ # There used to be code here to ensure that fcntl is not faked # but we can't do that cleanly any more - exc = raises(OSError, posix.fdopen, fd) - assert exc.value.errno == errno.EBADF + try: + fid = posix.fdopen(fd) + fid.read(10) + except IOError as e: + assert e.errno == errno.EBADF + else: + assert False, "using result of fdopen(fd) on closed file must raise" def test_fdopen_hackedbuiltins(self): "Same test, with __builtins__.file removed" @@ -331,8 +336,17 @@ path = self.path posix = self.posix fd = posix.open(path, posix.O_RDONLY) - exc = raises(OSError, posix.fdopen, fd, 'w') - assert str(exc.value) == "[Errno 22] Invalid argument" + # compatability issue - using Visual Studio 10 and above no + # longer raises on fid creation, only when _using_ fid + # win32 python2 raises IOError on flush(), win32 python3 raises OSError + try: + fid = posix.fdopen(fd, 'w') + fid.write('abc') + fid.flush() + except (OSError, IOError) as e: + assert e.errno in (9, 22) + else: + assert False, "expected OSError" posix.close(fd) # fd should not be closed def test_getcwd(self): diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -151,9 +151,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -187,20 +188,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -221,8 +236,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -296,7 +296,8 @@ def get_libc_name(): return rwin32.GetModuleFileName(get_libc_handle()) - assert "msvcr" in get_libc_name().lower(), \ + libc_name = get_libc_name().lower() + assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \ "Suspect msvcrt library: %s" % (get_libc_name(),) elif _MINGW: def get_libc_name(): diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,12 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3.3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { intptr_t osfhnd; @@ -95,6 +96,46 @@ errno = EBADF; return 0; } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: @@ -193,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): - # CPython Issue #11395, PyPy Issue #2636: the Windows console - # returns an error (12: not enough space error) on writing into - # stdout if stdout mode is binary and the length is greater than - # 66,000 bytes (or less, depending on heap usage). Can't easily - # test that, because we need 'fd' to be non-redirected... - count = 32767 - elif count > 0x7fffffff: - count = 0x7fffffff - return count -else: - def is_valid_fd(fd): - return 1 - - @enforceargs(int) - def validate_fd(fd): - pass - - def _bound_for_write(fd, count): - return count - -def closerange(fd_low, fd_high): - # this behaves like os.closerange() from Python 2.6. - for fd in xrange(fd_low, fd_high): - try: - if is_valid_fd(fd): - os.close(fd) - except OSError: - pass - if _WIN32: includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] @@ -258,11 +255,80 @@ if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'): includes.append('sys/ttycom.h') libraries = ['util'] + eci = ExternalCompilationInfo( includes=includes, libraries=libraries, ) +def external(name, args, result, compilation_info=eci, **kwds): + return rffi.llexternal(name, args, result, + compilation_info=compilation_info, **kwds) + + +if os.name == 'nt': + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + rffi.INT, compilation_info=errno_eci, + )) + c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", + [], rffi.VOIDP, compilation_info=errno_eci)) + c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph", + [rffi.VOIDP], lltype.Void, + compilation_info=errno_eci)) + + @enforceargs(int) + def _validate_fd(fd): + if not is_valid_fd(fd): + from errno import EBADF + raise OSError(EBADF, 'Bad file descriptor') + + class FdValidator(object): + + def __init__(self, fd): + _validate_fd(fd) + + def __enter__(self): + self.invalid_param_hndlr = c_enter_suppress_iph() + return self + + def __exit__(self, *args): + c_exit_suppress_iph(self.invalid_param_hndlr) + + def _bound_for_write(fd, count): + if count > 32767 and c_isatty(fd): + # CPython Issue #11395, PyPy Issue #2636: the Windows console + # returns an error (12: not enough space error) on writing into + # stdout if stdout mode is binary and the length is greater than + # 66,000 bytes (or less, depending on heap usage). Can't easily + # test that, because we need 'fd' to be non-redirected... + count = 32767 + elif count > 0x7fffffff: + count = 0x7fffffff + return count +else: + class FdValidator(object): + + def __init__(self, fd): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def _bound_for_write(fd, count): + return count + +def closerange(fd_low, fd_high): + # this behaves like os.closerange() from Python 2.6. + for fd in xrange(fd_low, fd_high): + try: + with FdValidator(fd): + os.close(fd) + except OSError: + pass + class CConfig: _compilation_info_ = eci SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET') @@ -315,10 +381,6 @@ config = rffi_platform.configure(CConfig) globals().update(config) -def external(name, args, result, compilation_info=eci, **kwds): - return rffi.llexternal(name, args, result, - compilation_info=compilation_info, **kwds) - # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. @@ -410,12 +472,12 @@ return result def _dup(fd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup(fd) - else: - res = c_dup_noninheritable(fd) - return res + with FdValidator(fd): + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res @replace_os_function('dup') def dup(fd, inheritable=True): @@ -424,12 +486,12 @@ @replace_os_function('dup2') def dup2(fd, newfd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup2(fd, newfd) - else: - res = c_dup2_noninheritable(fd, newfd) - handle_posix_error('dup2', res) + with FdValidator(fd): + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -457,25 +519,26 @@ def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - got = handle_posix_error('read', c_read(fd, void_buf, count)) - return buf.str(got) + with FdValidator(fd): + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + got = handle_posix_error('read', c_read(fd, void_buf, count)) + return buf.str(got) @replace_os_function('write') @enforceargs(int, None) def write(fd, data): count = len(data) - validate_fd(fd) - count = _bound_for_write(fd, count) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('write', c_write(fd, buf, count)) + with FdValidator(fd): + count = _bound_for_write(fd, count) + with rffi.scoped_nonmovingbuffer(data) as buf: + ret = c_write(fd, buf, count) + return handle_posix_error('write', ret) @replace_os_function('close') def close(fd): - validate_fd(fd) - handle_posix_error('close', c_close(fd)) + with FdValidator(fd): + handle_posix_error('close', c_close(fd)) c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, @@ -483,15 +546,15 @@ @replace_os_function('lseek') def lseek(fd, pos, how): - validate_fd(fd) - if SEEK_SET is not None: - if how == 0: - how = SEEK_SET - elif how == 1: - how = SEEK_CUR - elif how == 2: - how = SEEK_END - return handle_posix_error('lseek', c_lseek(fd, pos, how)) + with FdValidator(fd): + if SEEK_SET is not None: + if how == 0: + how = SEEK_SET + elif how == 1: + how = SEEK_CUR + elif how == 2: + how = SEEK_END + return handle_posix_error('lseek', c_lseek(fd, pos, how)) if not _WIN32: c_pread = external('pread', @@ -505,7 +568,6 @@ def pread(fd, count, offset): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) @@ -513,7 +575,6 @@ @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) - validate_fd(fd) with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) @@ -524,7 +585,6 @@ @enforceargs(int, None, None) def posix_fallocate(fd, offset, length): - validate_fd(fd) return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length)) if HAVE_FADVISE: @@ -546,7 +606,6 @@ @enforceargs(int, None, None, int) def posix_fadvise(fd, offset, length, advice): - validate_fd(fd) error = c_posix_fadvise(fd, offset, length, advice) error = widen(error) if error != 0: @@ -557,7 +616,6 @@ save_err=rffi.RFFI_SAVE_ERRNO) @enforceargs(int, None, None) def lockf(fd, cmd, length): - validate_fd(fd) return handle_posix_error('lockf', c_lockf(fd, cmd, length)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, @@ -571,18 +629,18 @@ @replace_os_function('ftruncate') def ftruncate(fd, length): - validate_fd(fd) - handle_posix_error('ftruncate', c_ftruncate(fd, length)) + with FdValidator(fd): + handle_posix_error('ftruncate', c_ftruncate(fd, length)) @replace_os_function('fsync') def fsync(fd): - validate_fd(fd) - handle_posix_error('fsync', c_fsync(fd)) + with FdValidator(fd): + handle_posix_error('fsync', c_fsync(fd)) @replace_os_function('fdatasync') def fdatasync(fd): - validate_fd(fd) - handle_posix_error('fdatasync', c_fdatasync(fd)) + with FdValidator(fd): + handle_posix_error('fdatasync', c_fdatasync(fd)) def sync(): c_sync() @@ -649,8 +707,8 @@ @replace_os_function('fchdir') def fchdir(fd): - validate_fd(fd) - handle_posix_error('fchdir', c_fchdir(fd)) + with FdValidator(fd): + handle_posix_error('fchdir', c_fchdir(fd)) @replace_os_function('access') @specialize.argtype(0) @@ -1090,9 +1148,9 @@ @replace_os_function('isatty') def isatty(fd): - if not is_valid_fd(fd): - return False - return c_isatty(fd) != 0 + with FdValidator(fd): + return c_isatty(fd) != 0 + return False c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -117,7 +117,7 @@ from rpython.translator.platform import host_factory static_platform = host_factory() if static_platform.name == 'msvc': - defines += ' PROCESS_QUERY_LIMITED_INFORMATION' + defines += ' PROCESS_QUERY_LIMITED_INFORMATION' for name in defines.split(): locals()[name] = rffi_platform.ConstantInteger(name) @@ -171,7 +171,7 @@ function, if that C function was declared with the flag llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). Functions without that flag don't change the saved LastError. - Alternatively, if the function was declared + Alternatively, if the function was declared RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, then the value of the C-level WSAGetLastError() is saved instead (into the same "saved alt LastError" variable). @@ -218,9 +218,9 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) def get_osfhandle(fd): - from rpython.rlib.rposix import validate_fd - validate_fd(fd) - handle = _get_osfhandle(fd) + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + handle = _get_osfhandle(fd) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle @@ -231,45 +231,10 @@ in the dict.""" # Prior to Visual Studio 8, the MSVCRT dll doesn't export the # _dosmaperr() function, which is available only when compiled - # against the static CRT library. - from rpython.translator.platform import host_factory - static_platform = host_factory() - if static_platform.name == 'msvc': - static_platform.cflags = ['/MT'] # static CRT - static_platform.version = 0 # no manifest - cfile = udir.join('dosmaperr.c') - cfile.write(r''' - #include - #include - #include - #ifdef __GNUC__ - #define _dosmaperr mingw_dosmaperr - #endif - int main() - { - int i; - for(i=1; i < 65000; i++) { - _dosmaperr(i); - if (errno == EINVAL) { - /* CPython issue #12802 */ - if (i == ERROR_DIRECTORY) - errno = ENOTDIR; - else - continue; - } - printf("%d\t%d\n", i, errno); - } - return 0; - }''') - try: - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - except (CompilationError, WindowsError): - # Fallback for the mingw32 compiler - assert static_platform.name == 'mingw32' - errors = { + # against the static CRT library. After Visual Studio 9, this + # private function seems to be gone, so use a static map, from + # CPython PC/errmap.h + errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, @@ -279,12 +244,8 @@ 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, - 202: 8, 206: 2, 215: 11, 267: 20, 1816: 12, + 202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, } - else: - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) return errors, errno.EINVAL # A bit like strerror... @@ -299,7 +260,7 @@ buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), @@ -330,7 +291,7 @@ buf[0] = lltype.nullptr(rffi.CWCHARP.TO) try: msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -15,19 +15,6 @@ py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) -def test_get_osfhandle_raising(): - #try to test what kind of exception get_osfhandle raises w/out fd validation - py.test.skip('Crashes python') - fid = open(str(udir.join('validate_test.txt')), 'w') - fd = fid.fileno() - fid.close() - def validate_fd(fd): - return 1 - _validate_fd = rwin32.validate_fd - rwin32.validate_fd = validate_fd - raises(WindowsError, rwin32.get_osfhandle, fd) - rwin32.validate_fd = _validate_fd - def test_open_process(): pid = rwin32.GetCurrentProcessId() assert pid != 0 diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py new file mode 100644 --- /dev/null +++ b/rpython/tool/setuptools_msvc.py @@ -0,0 +1,1308 @@ +""" +Improved support for Microsoft Visual C++ compilers. + +Known supported compilers: +-------------------------- +Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + +Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + +Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) +""" + +# copied as-is from python3 +# modified to: +# - commented out monkey patching +# - lookup() returns str not unicode + +import os +import sys +import platform +import itertools +import distutils.errors +from pkg_resources.extern.packaging.version import LegacyVersion + +from setuptools.extern.six.moves import filterfalse + +#from .monkey import get_unpatched + +if platform.system() == 'Windows': + from setuptools.extern.six.moves import winreg + safe_env = os.environ +else: + """ + Mock winreg and environ so the module can be imported + on this platform. + """ + + class winreg: + HKEY_USERS = None + HKEY_CURRENT_USER = None + HKEY_LOCAL_MACHINE = None + HKEY_CLASSES_ROOT = None + + safe_env = dict() + +_msvc9_suppress_errors = ( + # msvc9compiler isn't available on some platforms + ImportError, + + # msvc9compiler raises DistutilsPlatformError in some + # environments. See #1118. + distutils.errors.DistutilsPlatformError, +) + +try: + from distutils.msvc9compiler import Reg +except _msvc9_suppress_errors: + pass + + +def msvc9_find_vcvarsall(version): + """ + Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone + compiler build for Python (VCForPython). Fall back to original behavior + when the standalone compiler is not available. + + Redirect the path of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + + Parameters + ---------- + version: float + Required Microsoft Visual C++ version. + + Return + ------ + vcvarsall.bat path: str + """ + VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) + try: + # Per-user installs register the compiler path here + productdir = Reg.get_value(key, "installdir") + except KeyError: + try: + # All-user installs on a 64-bit system register here + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") + except KeyError: + productdir = None + + if productdir: + vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + + return get_unpatched(msvc9_find_vcvarsall)(version) + + +def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): + """ + Patched "distutils.msvc9compiler.query_vcvarsall" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + + Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + + Parameters + ---------- + ver: float + Required Microsoft Visual C++ version. + arch: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environement from vcvarsall.bat (Classical way) + #try: + # orig = get_unpatched(msvc9_query_vcvarsall) + # return orig(ver, arch, *args, **kwargs) + #except distutils.errors.DistutilsPlatformError: + # # Pass error if Vcvarsall.bat is missing + # pass + #except ValueError: + # # Pass error if environment not set after executing vcvarsall.bat + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(arch, ver).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, ver, arch) + raise + + +def msvc14_get_vc_env(plat_spec): + """ + Patched "distutils._msvccompiler._get_vc_env" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) + + Parameters + ---------- + plat_spec: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environment from vcvarsall.bat (Classical way) + #try: + # return get_unpatched(msvc14_get_vc_env)(plat_spec) + #except distutils.errors.DistutilsPlatformError: + # Pass error Vcvarsall.bat is missing + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, 14.0) + raise + + +def msvc14_gen_lib_options(*args, **kwargs): + """ + Patched "distutils._msvccompiler.gen_lib_options" for fix + compatibility between "numpy.distutils" and "distutils._msvccompiler" + (for Numpy < 1.11.2) + """ + if "numpy.distutils" in sys.modules: + import numpy as np + if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'): + return np.distutils.ccompiler.gen_lib_options(*args, **kwargs) + return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs) + + +def _augment_exception(exc, version, arch=''): + """ + Add details to the exception message to help guide the user + as to what action will resolve it. + """ + # Error if MSVC++ directory not found or environment not set + message = exc.args[0] + + if "vcvarsall" in message.lower() or "visual c" in message.lower(): + # Special error message if MSVC++ not installed + tmpl = 'Microsoft Visual C++ {version:0.1f} is required.' + message = tmpl.format(**locals()) + msdownload = 'www.microsoft.com/download/details.aspx?id=%d' + if version == 9.0: + if arch.lower().find('ia64') > -1: + # For VC++ 9.0, if IA64 support is needed, redirect user + # to Windows SDK 7.0 + message += ' Get it with "Microsoft Windows SDK 7.0": ' + message += msdownload % 3138 + else: + # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : + # This redirection link is maintained by Microsoft. + # Contact vspython at microsoft.com if it needs updating. + message += ' Get it from http://aka.ms/vcpython27' + elif version == 10.0: + # For VC++ 10.0 Redirect user to Windows SDK 7.1 + message += ' Get it with "Microsoft Windows SDK 7.1": ' + message += msdownload % 8279 + elif version >= 14.0: + # For VC++ 14.0 Redirect user to Visual C++ Build Tools + message += (' Get it with "Microsoft Visual C++ Build Tools": ' + r'http://landinghub.visualstudio.com/' + 'visual-cpp-build-tools') + + exc.args = (message, ) + + +class PlatformInfo: + """ + Current and Target Architectures informations. + + Parameters + ---------- + arch: str + Target architecture. + """ + current_cpu = safe_env.get('processor_architecture', '').lower() + + def __init__(self, arch): + self.arch = arch.lower().replace('x64', 'amd64') + + @property + def target_cpu(self): + return self.arch[self.arch.find('_') + 1:] + + def target_is_x86(self): + return self.target_cpu == 'x86' + + def current_is_x86(self): + return self.current_cpu == 'x86' + + def current_dir(self, hidex86=False, x64=False): + """ + Current platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\target', or '' (see hidex86 parameter) + """ + return ( + '' if (self.current_cpu == 'x86' and hidex86) else + r'\x64' if (self.current_cpu == 'amd64' and x64) else + r'\%s' % self.current_cpu + ) + + def target_dir(self, hidex86=False, x64=False): + r""" + Target platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\current', or '' (see hidex86 parameter) + """ + return ( + '' if (self.target_cpu == 'x86' and hidex86) else + r'\x64' if (self.target_cpu == 'amd64' and x64) else + r'\%s' % self.target_cpu + ) + + def cross_dir(self, forcex86=False): + r""" + Cross platform specific subfolder. + + Parameters + ---------- + forcex86: bool + Use 'x86' as current architecture even if current acritecture is + not x86. + + Return + ------ + subfolder: str + '' if target architecture is current architecture, + '\current_target' if not. + """ + current = 'x86' if forcex86 else self.current_cpu + return ( + '' if self.target_cpu == current else + self.target_dir().replace('\\', '\\%s_' % current) + ) + + +class RegistryInfo: + """ + Microsoft Visual Studio related registry informations. + + Parameters + ---------- + platform_info: PlatformInfo + "PlatformInfo" instance. + """ + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + + def __init__(self, platform_info): + self.pi = platform_info + + @property + def visualstudio(self): + """ + Microsoft Visual Studio root registry key. + """ + return 'VisualStudio' + + @property + def sxs(self): + """ + Microsoft Visual Studio SxS registry key. + """ + return os.path.join(self.visualstudio, 'SxS') + + @property + def vc(self): + """ + Microsoft Visual C++ VC7 registry key. + """ + return os.path.join(self.sxs, 'VC7') + + @property + def vs(self): + """ + Microsoft Visual Studio VS7 registry key. + """ + return os.path.join(self.sxs, 'VS7') + + @property + def vc_for_python(self): + """ + Microsoft Visual C++ for Python registry key. + """ + return r'DevDiv\VCForPython' + + @property + def microsoft_sdk(self): + """ + Microsoft SDK registry key. + """ + return 'Microsoft SDKs' + + @property + def windows_sdk(self): + """ + Microsoft Windows/Platform SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'Windows') + + @property + def netfx_sdk(self): + """ + Microsoft .NET Framework SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'NETFXSDK') + + @property + def windows_kits_roots(self): + """ + Microsoft Windows Kits Roots registry key. + """ + return r'Windows Kits\Installed Roots' + + def microsoft(self, key, x86=False): + """ + Return key in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + x86: str + Force x86 software registry. + + Return + ------ + str: value + """ + node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node' + return os.path.join('Software', node64, 'Microsoft', key) + + def lookup(self, key, name): + """ + Look for values in registry in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + name: str + Value name to find. + + Return + ------ + str: value + """ + KEY_READ = winreg.KEY_READ + openkey = winreg.OpenKey + ms = self.microsoft + for hkey in self.HKEYS: + try: + bkey = openkey(hkey, ms(key), 0, KEY_READ) + except (OSError, IOError): + if not self.pi.current_is_x86(): + try: + bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + except (OSError, IOError): + continue + else: + continue + try: + # modified to return str for python2 + return winreg.QueryValueEx(bkey, name)[0].encode('utf8') + except (OSError, IOError): + pass + + +class SystemInfo: + """ + Microsoft Windows and Visual Studio related system inormations. + + Parameters + ---------- + registry_info: RegistryInfo + "RegistryInfo" instance. + vc_ver: float + Required Microsoft Visual C++ version. + """ + + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + WinDir = safe_env.get('WinDir', '') + ProgramFiles = safe_env.get('ProgramFiles', '') + ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + + def __init__(self, registry_info, vc_ver=None): + self.ri = registry_info + self.pi = self.ri.pi + self.vc_ver = vc_ver or self._find_latest_available_vc_ver() + + def _find_latest_available_vc_ver(self): + try: + return self.find_available_vc_vers()[-1] + except IndexError: + err = 'No Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + def find_available_vc_vers(self): + """ + Find all available Microsoft Visual C++ versions. + """ + ms = self.ri.microsoft + vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) + vc_vers = [] + for hkey in self.ri.HKEYS: + for key in vckeys: + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + try: + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + for i in range(subkeys): + try: + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + return sorted(vc_vers) + + @property + def VSInstallDir(self): + """ + Microsoft Visual Studio directory. + """ + # Default path + name = 'Microsoft Visual Studio %0.1f' % self.vc_ver + default = os.path.join(self.ProgramFilesx86, name) + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + + @property + def VCInstallDir(self): + """ + Microsoft Visual C++ directory. + """ + self.VSInstallDir + + guess_vc = self._guess_vc() or self._guess_vc_legacy() + + # Try to get "VC++ for Python" path from registry as default path + reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc + + # Try to get path from registry, if fail use default path + path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + + if not os.path.isdir(path): + msg = 'Microsoft Visual C++ directory not found' + raise distutils.errors.DistutilsPlatformError(msg) + + return path + + def _guess_vc(self): + """ + Locate Visual C for 2017 + """ + if self.vc_ver <= 14.0: + return + + default = r'VC\Tools\MSVC' + guess_vc = os.path.join(self.VSInstallDir, default) + # Subdir with VC exact version as name + try: + vc_exact_ver = os.listdir(guess_vc)[-1] + return os.path.join(guess_vc, vc_exact_ver) + except (OSError, IOError, IndexError): + pass + + def _guess_vc_legacy(self): + """ + Locate Visual C for versions prior to 2017 + """ + default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver + return os.path.join(self.ProgramFilesx86, default) + + @property + def WindowsSdkVersion(self): + """ + Microsoft Windows SDK versions for specified MSVC++ version. + """ + if self.vc_ver <= 9.0: + return ('7.0', '6.1', '6.0a') + elif self.vc_ver == 10.0: + return ('7.1', '7.0a') + elif self.vc_ver == 11.0: + return ('8.0', '8.0a') + elif self.vc_ver == 12.0: + return ('8.1', '8.1a') + elif self.vc_ver >= 14.0: + return ('10.0', '8.1') + + @property + def WindowsSdkLastVersion(self): + """ + Microsoft Windows SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.WindowsSdkDir, 'lib')) + + @property + def WindowsSdkDir(self): + """ + Microsoft Windows SDK directory. + """ + sdkdir = '' + for ver in self.WindowsSdkVersion: + # Try to get it from registry + loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + sdkdir = self.ri.lookup(loc, 'installationfolder') + if sdkdir: + break + if not sdkdir or not os.path.isdir(sdkdir): + # Try to get "VC++ for Python" version from registry + path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + install_base = self.ri.lookup(path, 'installdir') + if install_base: + sdkdir = os.path.join(install_base, 'WinSDK') + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default new path + for ver in self.WindowsSdkVersion: + intver = ver[:ver.rfind('.')] + path = r'Microsoft SDKs\Windows Kits\%s' % (intver) + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default old path + for ver in self.WindowsSdkVersion: + path = r'Microsoft SDKs\Windows\v%s' % ver + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir: + # If fail, use Platform SDK + sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + return sdkdir + + @property + def WindowsSDKExecutablePath(self): + """ + Microsoft Windows SDK executable directory. + """ + # Find WinSDK NetFx Tools registry dir name + if self.vc_ver <= 11.0: + netfxver = 35 + arch = '' + else: + netfxver = 40 + hidex86 = True if self.vc_ver <= 12.0 else False + arch = self.pi.current_dir(x64=True, hidex86=hidex86) + fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) + + # liste all possibles registry paths + regpaths = [] + if self.vc_ver >= 14.0: + for ver in self.NetFxSdkVersion: + regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + + for ver in self.WindowsSdkVersion: + regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + + # Return installation folder from the more recent path + for path in regpaths: + execpath = self.ri.lookup(path, 'installationfolder') + if execpath: + break + return execpath + + @property + def FSharpInstallDir(self): + """ + Microsoft Visual F# directory. + """ + path = r'%0.1f\Setup\F#' % self.vc_ver + path = os.path.join(self.ri.visualstudio, path) + return self.ri.lookup(path, 'productdir') or '' + + @property + def UniversalCRTSdkDir(self): + """ + Microsoft Universal CRT SDK directory. + """ + # Set Kit Roots versions for specified MSVC++ version + if self.vc_ver >= 14.0: + vers = ('10', '81') + else: + vers = () + + # Find path of the more recent Kit + for ver in vers: + sdkdir = self.ri.lookup(self.ri.windows_kits_roots, + 'kitsroot%s' % ver) + if sdkdir: + break + return sdkdir or '' + + @property + def UniversalCRTSdkLastVersion(self): + """ + Microsoft Universal C Runtime SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.UniversalCRTSdkDir, 'lib')) + + @property + def NetFxSdkVersion(self): + """ + Microsoft .NET Framework SDK versions. + """ + # Set FxSdk versions for specified MSVC++ version + if self.vc_ver >= 14.0: + return ('4.6.1', '4.6') + else: + return () + + @property + def NetFxSdkDir(self): + """ + Microsoft .NET Framework SDK directory. + """ + for ver in self.NetFxSdkVersion: + loc = os.path.join(self.ri.netfx_sdk, ver) + sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder') From pypy.commits at gmail.com Mon Feb 12 14:27:12 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:12 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5a81ea90.c8c51c0a.5b14f.dfcd@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93814:69009f281e55 Date: 2018-02-12 14:24 -0500 http://bitbucket.org/pypy/pypy/changeset/69009f281e55/ Log: merge default into py3.5 diff too long, truncating to 2000 out of 2875 lines diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -40,3 +40,7 @@ .. branch: memory-accounting Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,6 +1,7 @@ import py import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be) from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp @@ -51,7 +52,10 @@ py.test.raises(Hit, decode_utf8, space, "\xed\xb0\x80") py.test.raises(Hit, decode_utf8, space, "\xed\xa0\x80\xed\xb0\x80") got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] def test_decode_utf8_allow_surrogates(): sp = FakeSpace() diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -105,14 +105,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -589,12 +589,17 @@ assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys import codecs sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' assert b'\x00\x60\x80'.decode( diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1202,12 +1202,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -300,6 +300,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -31,7 +31,7 @@ from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -97,25 +97,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,6 +1,5 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import errno @@ -94,6 +93,10 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen + class uname_result(metaclass=structseqtype): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -17,6 +17,7 @@ USEMODULES += ['fcntl', 'select', '_posixsubprocess', '_socket'] else: USEMODULES += ['_rawffi', 'thread'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -221,9 +221,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -332,20 +333,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { return _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -359,8 +374,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -296,7 +296,8 @@ def get_libc_name(): return rwin32.GetModuleFileName(get_libc_handle()) - assert "msvcr" in get_libc_name().lower(), \ + libc_name = get_libc_name().lower() + assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \ "Suspect msvcrt library: %s" % (get_libc_name(),) elif _MINGW: def get_libc_name(): diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,12 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3.3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { intptr_t osfhnd; @@ -95,6 +96,46 @@ errno = EBADF; return 0; } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: @@ -193,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): - # CPython Issue #11395, PyPy Issue #2636: the Windows console - # returns an error (12: not enough space error) on writing into - # stdout if stdout mode is binary and the length is greater than - # 66,000 bytes (or less, depending on heap usage). Can't easily - # test that, because we need 'fd' to be non-redirected... - count = 32767 - elif count > 0x7fffffff: - count = 0x7fffffff - return count -else: - def is_valid_fd(fd): - return 1 - - @enforceargs(int) - def validate_fd(fd): - pass - - def _bound_for_write(fd, count): - return count - -def closerange(fd_low, fd_high): - # this behaves like os.closerange() from Python 2.6. - for fd in xrange(fd_low, fd_high): - try: - if is_valid_fd(fd): - os.close(fd) - except OSError: - pass - if _WIN32: includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] @@ -258,11 +255,80 @@ if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'): includes.append('sys/ttycom.h') libraries = ['util'] + eci = ExternalCompilationInfo( includes=includes, libraries=libraries, ) +def external(name, args, result, compilation_info=eci, **kwds): + return rffi.llexternal(name, args, result, + compilation_info=compilation_info, **kwds) + + +if os.name == 'nt': + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + rffi.INT, compilation_info=errno_eci, + )) + c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", + [], rffi.VOIDP, compilation_info=errno_eci)) + c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph", + [rffi.VOIDP], lltype.Void, + compilation_info=errno_eci)) + + @enforceargs(int) + def _validate_fd(fd): + if not is_valid_fd(fd): + from errno import EBADF + raise OSError(EBADF, 'Bad file descriptor') + + class FdValidator(object): + + def __init__(self, fd): + _validate_fd(fd) + + def __enter__(self): + self.invalid_param_hndlr = c_enter_suppress_iph() + return self + + def __exit__(self, *args): + c_exit_suppress_iph(self.invalid_param_hndlr) + + def _bound_for_write(fd, count): + if count > 32767 and c_isatty(fd): + # CPython Issue #11395, PyPy Issue #2636: the Windows console + # returns an error (12: not enough space error) on writing into + # stdout if stdout mode is binary and the length is greater than + # 66,000 bytes (or less, depending on heap usage). Can't easily + # test that, because we need 'fd' to be non-redirected... + count = 32767 + elif count > 0x7fffffff: + count = 0x7fffffff + return count +else: + class FdValidator(object): + + def __init__(self, fd): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def _bound_for_write(fd, count): + return count + +def closerange(fd_low, fd_high): + # this behaves like os.closerange() from Python 2.6. + for fd in xrange(fd_low, fd_high): + try: + with FdValidator(fd): + os.close(fd) + except OSError: + pass + class CConfig: _compilation_info_ = eci SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET') @@ -315,10 +381,6 @@ config = rffi_platform.configure(CConfig) globals().update(config) -def external(name, args, result, compilation_info=eci, **kwds): - return rffi.llexternal(name, args, result, - compilation_info=compilation_info, **kwds) - # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. @@ -410,12 +472,12 @@ return result def _dup(fd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup(fd) - else: - res = c_dup_noninheritable(fd) - return res + with FdValidator(fd): + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res @replace_os_function('dup') def dup(fd, inheritable=True): @@ -424,12 +486,12 @@ @replace_os_function('dup2') def dup2(fd, newfd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup2(fd, newfd) - else: - res = c_dup2_noninheritable(fd, newfd) - handle_posix_error('dup2', res) + with FdValidator(fd): + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -457,25 +519,26 @@ def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - got = handle_posix_error('read', c_read(fd, void_buf, count)) - return buf.str(got) + with FdValidator(fd): + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + got = handle_posix_error('read', c_read(fd, void_buf, count)) + return buf.str(got) @replace_os_function('write') @enforceargs(int, None) def write(fd, data): count = len(data) - validate_fd(fd) - count = _bound_for_write(fd, count) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('write', c_write(fd, buf, count)) + with FdValidator(fd): + count = _bound_for_write(fd, count) + with rffi.scoped_nonmovingbuffer(data) as buf: + ret = c_write(fd, buf, count) + return handle_posix_error('write', ret) @replace_os_function('close') def close(fd): - validate_fd(fd) - handle_posix_error('close', c_close(fd)) + with FdValidator(fd): + handle_posix_error('close', c_close(fd)) c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, @@ -483,15 +546,15 @@ @replace_os_function('lseek') def lseek(fd, pos, how): - validate_fd(fd) - if SEEK_SET is not None: - if how == 0: - how = SEEK_SET - elif how == 1: - how = SEEK_CUR - elif how == 2: - how = SEEK_END - return handle_posix_error('lseek', c_lseek(fd, pos, how)) + with FdValidator(fd): + if SEEK_SET is not None: + if how == 0: + how = SEEK_SET + elif how == 1: + how = SEEK_CUR + elif how == 2: + how = SEEK_END + return handle_posix_error('lseek', c_lseek(fd, pos, how)) if not _WIN32: c_pread = external('pread', @@ -505,7 +568,6 @@ def pread(fd, count, offset): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) @@ -513,7 +575,6 @@ @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) - validate_fd(fd) with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) @@ -524,7 +585,6 @@ @enforceargs(int, None, None) def posix_fallocate(fd, offset, length): - validate_fd(fd) return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length)) if HAVE_FADVISE: @@ -546,7 +606,6 @@ @enforceargs(int, None, None, int) def posix_fadvise(fd, offset, length, advice): - validate_fd(fd) error = c_posix_fadvise(fd, offset, length, advice) error = widen(error) if error != 0: @@ -557,7 +616,6 @@ save_err=rffi.RFFI_SAVE_ERRNO) @enforceargs(int, None, None) def lockf(fd, cmd, length): - validate_fd(fd) return handle_posix_error('lockf', c_lockf(fd, cmd, length)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, @@ -571,18 +629,18 @@ @replace_os_function('ftruncate') def ftruncate(fd, length): - validate_fd(fd) - handle_posix_error('ftruncate', c_ftruncate(fd, length)) + with FdValidator(fd): + handle_posix_error('ftruncate', c_ftruncate(fd, length)) @replace_os_function('fsync') def fsync(fd): - validate_fd(fd) - handle_posix_error('fsync', c_fsync(fd)) + with FdValidator(fd): + handle_posix_error('fsync', c_fsync(fd)) @replace_os_function('fdatasync') def fdatasync(fd): - validate_fd(fd) - handle_posix_error('fdatasync', c_fdatasync(fd)) + with FdValidator(fd): + handle_posix_error('fdatasync', c_fdatasync(fd)) def sync(): c_sync() @@ -649,8 +707,8 @@ @replace_os_function('fchdir') def fchdir(fd): - validate_fd(fd) - handle_posix_error('fchdir', c_fchdir(fd)) + with FdValidator(fd): + handle_posix_error('fchdir', c_fchdir(fd)) @replace_os_function('access') @specialize.argtype(0) @@ -1090,9 +1148,9 @@ @replace_os_function('isatty') def isatty(fd): - if not is_valid_fd(fd): - return False - return c_isatty(fd) != 0 + with FdValidator(fd): + return c_isatty(fd) != 0 + return False c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -117,7 +117,7 @@ from rpython.translator.platform import host_factory static_platform = host_factory() if static_platform.name == 'msvc': - defines += ' PROCESS_QUERY_LIMITED_INFORMATION' + defines += ' PROCESS_QUERY_LIMITED_INFORMATION' for name in defines.split(): locals()[name] = rffi_platform.ConstantInteger(name) @@ -171,7 +171,7 @@ function, if that C function was declared with the flag llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). Functions without that flag don't change the saved LastError. - Alternatively, if the function was declared + Alternatively, if the function was declared RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, then the value of the C-level WSAGetLastError() is saved instead (into the same "saved alt LastError" variable). @@ -218,9 +218,9 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) def get_osfhandle(fd): - from rpython.rlib.rposix import validate_fd - validate_fd(fd) - handle = _get_osfhandle(fd) + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + handle = _get_osfhandle(fd) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle @@ -231,45 +231,10 @@ in the dict.""" # Prior to Visual Studio 8, the MSVCRT dll doesn't export the # _dosmaperr() function, which is available only when compiled - # against the static CRT library. - from rpython.translator.platform import host_factory - static_platform = host_factory() - if static_platform.name == 'msvc': - static_platform.cflags = ['/MT'] # static CRT - static_platform.version = 0 # no manifest - cfile = udir.join('dosmaperr.c') - cfile.write(r''' - #include - #include - #include - #ifdef __GNUC__ - #define _dosmaperr mingw_dosmaperr - #endif - int main() - { - int i; - for(i=1; i < 65000; i++) { - _dosmaperr(i); - if (errno == EINVAL) { - /* CPython issue #12802 */ - if (i == ERROR_DIRECTORY) - errno = ENOTDIR; - else - continue; - } - printf("%d\t%d\n", i, errno); - } - return 0; - }''') - try: - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - except (CompilationError, WindowsError): - # Fallback for the mingw32 compiler - assert static_platform.name == 'mingw32' - errors = { + # against the static CRT library. After Visual Studio 9, this + # private function seems to be gone, so use a static map, from + # CPython PC/errmap.h + errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, @@ -279,12 +244,8 @@ 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, - 202: 8, 206: 2, 215: 11, 267: 20, 1816: 12, + 202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, } - else: - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) return errors, errno.EINVAL # A bit like strerror... @@ -299,7 +260,7 @@ buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), @@ -330,7 +291,7 @@ buf[0] = lltype.nullptr(rffi.CWCHARP.TO) try: msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -15,19 +15,6 @@ py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) -def test_get_osfhandle_raising(): - #try to test what kind of exception get_osfhandle raises w/out fd validation - py.test.skip('Crashes python') - fid = open(str(udir.join('validate_test.txt')), 'w') - fd = fid.fileno() - fid.close() - def validate_fd(fd): - return 1 - _validate_fd = rwin32.validate_fd - rwin32.validate_fd = validate_fd - raises(WindowsError, rwin32.get_osfhandle, fd) - rwin32.validate_fd = _validate_fd - def test_open_process(): pid = rwin32.GetCurrentProcessId() assert pid != 0 diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py new file mode 100644 --- /dev/null +++ b/rpython/tool/setuptools_msvc.py @@ -0,0 +1,1308 @@ +""" +Improved support for Microsoft Visual C++ compilers. + +Known supported compilers: +-------------------------- +Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + +Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + +Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) +""" + +# copied as-is from python3 +# modified to: +# - commented out monkey patching +# - lookup() returns str not unicode + +import os +import sys +import platform +import itertools +import distutils.errors +from pkg_resources.extern.packaging.version import LegacyVersion + +from setuptools.extern.six.moves import filterfalse + +#from .monkey import get_unpatched + +if platform.system() == 'Windows': + from setuptools.extern.six.moves import winreg + safe_env = os.environ +else: + """ + Mock winreg and environ so the module can be imported + on this platform. + """ + + class winreg: + HKEY_USERS = None + HKEY_CURRENT_USER = None + HKEY_LOCAL_MACHINE = None + HKEY_CLASSES_ROOT = None + + safe_env = dict() + +_msvc9_suppress_errors = ( + # msvc9compiler isn't available on some platforms + ImportError, + + # msvc9compiler raises DistutilsPlatformError in some + # environments. See #1118. + distutils.errors.DistutilsPlatformError, +) + +try: + from distutils.msvc9compiler import Reg +except _msvc9_suppress_errors: + pass + + +def msvc9_find_vcvarsall(version): + """ + Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone + compiler build for Python (VCForPython). Fall back to original behavior + when the standalone compiler is not available. + + Redirect the path of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + + Parameters + ---------- + version: float + Required Microsoft Visual C++ version. + + Return + ------ + vcvarsall.bat path: str + """ + VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) + try: + # Per-user installs register the compiler path here + productdir = Reg.get_value(key, "installdir") + except KeyError: + try: + # All-user installs on a 64-bit system register here + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") + except KeyError: + productdir = None + + if productdir: + vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + + return get_unpatched(msvc9_find_vcvarsall)(version) + + +def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): + """ + Patched "distutils.msvc9compiler.query_vcvarsall" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + + Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + + Parameters + ---------- + ver: float + Required Microsoft Visual C++ version. + arch: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environement from vcvarsall.bat (Classical way) + #try: + # orig = get_unpatched(msvc9_query_vcvarsall) + # return orig(ver, arch, *args, **kwargs) + #except distutils.errors.DistutilsPlatformError: + # # Pass error if Vcvarsall.bat is missing + # pass + #except ValueError: + # # Pass error if environment not set after executing vcvarsall.bat + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(arch, ver).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, ver, arch) + raise + + +def msvc14_get_vc_env(plat_spec): + """ + Patched "distutils._msvccompiler._get_vc_env" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) + + Parameters + ---------- + plat_spec: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environment from vcvarsall.bat (Classical way) + #try: + # return get_unpatched(msvc14_get_vc_env)(plat_spec) + #except distutils.errors.DistutilsPlatformError: + # Pass error Vcvarsall.bat is missing + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, 14.0) + raise + + +def msvc14_gen_lib_options(*args, **kwargs): + """ + Patched "distutils._msvccompiler.gen_lib_options" for fix + compatibility between "numpy.distutils" and "distutils._msvccompiler" + (for Numpy < 1.11.2) + """ + if "numpy.distutils" in sys.modules: + import numpy as np + if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'): + return np.distutils.ccompiler.gen_lib_options(*args, **kwargs) + return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs) + + +def _augment_exception(exc, version, arch=''): + """ + Add details to the exception message to help guide the user + as to what action will resolve it. + """ + # Error if MSVC++ directory not found or environment not set + message = exc.args[0] + + if "vcvarsall" in message.lower() or "visual c" in message.lower(): + # Special error message if MSVC++ not installed + tmpl = 'Microsoft Visual C++ {version:0.1f} is required.' + message = tmpl.format(**locals()) + msdownload = 'www.microsoft.com/download/details.aspx?id=%d' + if version == 9.0: + if arch.lower().find('ia64') > -1: + # For VC++ 9.0, if IA64 support is needed, redirect user + # to Windows SDK 7.0 + message += ' Get it with "Microsoft Windows SDK 7.0": ' + message += msdownload % 3138 + else: + # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : + # This redirection link is maintained by Microsoft. + # Contact vspython at microsoft.com if it needs updating. + message += ' Get it from http://aka.ms/vcpython27' + elif version == 10.0: + # For VC++ 10.0 Redirect user to Windows SDK 7.1 + message += ' Get it with "Microsoft Windows SDK 7.1": ' + message += msdownload % 8279 + elif version >= 14.0: + # For VC++ 14.0 Redirect user to Visual C++ Build Tools + message += (' Get it with "Microsoft Visual C++ Build Tools": ' + r'http://landinghub.visualstudio.com/' + 'visual-cpp-build-tools') + + exc.args = (message, ) + + +class PlatformInfo: + """ + Current and Target Architectures informations. + + Parameters + ---------- + arch: str + Target architecture. + """ + current_cpu = safe_env.get('processor_architecture', '').lower() + + def __init__(self, arch): + self.arch = arch.lower().replace('x64', 'amd64') + + @property + def target_cpu(self): + return self.arch[self.arch.find('_') + 1:] + + def target_is_x86(self): + return self.target_cpu == 'x86' + + def current_is_x86(self): + return self.current_cpu == 'x86' + + def current_dir(self, hidex86=False, x64=False): + """ + Current platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\target', or '' (see hidex86 parameter) + """ + return ( + '' if (self.current_cpu == 'x86' and hidex86) else + r'\x64' if (self.current_cpu == 'amd64' and x64) else + r'\%s' % self.current_cpu + ) + + def target_dir(self, hidex86=False, x64=False): + r""" + Target platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\current', or '' (see hidex86 parameter) + """ + return ( + '' if (self.target_cpu == 'x86' and hidex86) else + r'\x64' if (self.target_cpu == 'amd64' and x64) else + r'\%s' % self.target_cpu + ) + + def cross_dir(self, forcex86=False): + r""" + Cross platform specific subfolder. + + Parameters + ---------- + forcex86: bool + Use 'x86' as current architecture even if current acritecture is + not x86. + + Return + ------ + subfolder: str + '' if target architecture is current architecture, + '\current_target' if not. + """ + current = 'x86' if forcex86 else self.current_cpu + return ( + '' if self.target_cpu == current else + self.target_dir().replace('\\', '\\%s_' % current) + ) + + +class RegistryInfo: + """ + Microsoft Visual Studio related registry informations. + + Parameters + ---------- + platform_info: PlatformInfo + "PlatformInfo" instance. + """ + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + + def __init__(self, platform_info): + self.pi = platform_info + + @property + def visualstudio(self): + """ + Microsoft Visual Studio root registry key. + """ + return 'VisualStudio' + + @property + def sxs(self): + """ + Microsoft Visual Studio SxS registry key. + """ + return os.path.join(self.visualstudio, 'SxS') + + @property + def vc(self): + """ + Microsoft Visual C++ VC7 registry key. + """ + return os.path.join(self.sxs, 'VC7') + + @property + def vs(self): + """ + Microsoft Visual Studio VS7 registry key. + """ + return os.path.join(self.sxs, 'VS7') + + @property + def vc_for_python(self): + """ + Microsoft Visual C++ for Python registry key. + """ + return r'DevDiv\VCForPython' + + @property + def microsoft_sdk(self): + """ + Microsoft SDK registry key. + """ + return 'Microsoft SDKs' + + @property + def windows_sdk(self): + """ + Microsoft Windows/Platform SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'Windows') + + @property + def netfx_sdk(self): + """ + Microsoft .NET Framework SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'NETFXSDK') + + @property + def windows_kits_roots(self): + """ + Microsoft Windows Kits Roots registry key. + """ + return r'Windows Kits\Installed Roots' + + def microsoft(self, key, x86=False): + """ + Return key in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + x86: str + Force x86 software registry. + + Return + ------ + str: value + """ + node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node' + return os.path.join('Software', node64, 'Microsoft', key) + + def lookup(self, key, name): + """ + Look for values in registry in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + name: str + Value name to find. + + Return + ------ + str: value + """ + KEY_READ = winreg.KEY_READ + openkey = winreg.OpenKey + ms = self.microsoft + for hkey in self.HKEYS: + try: + bkey = openkey(hkey, ms(key), 0, KEY_READ) + except (OSError, IOError): + if not self.pi.current_is_x86(): + try: + bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + except (OSError, IOError): + continue + else: + continue + try: + # modified to return str for python2 + return winreg.QueryValueEx(bkey, name)[0].encode('utf8') + except (OSError, IOError): + pass + + +class SystemInfo: + """ + Microsoft Windows and Visual Studio related system inormations. + + Parameters + ---------- + registry_info: RegistryInfo + "RegistryInfo" instance. + vc_ver: float + Required Microsoft Visual C++ version. + """ + + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + WinDir = safe_env.get('WinDir', '') + ProgramFiles = safe_env.get('ProgramFiles', '') + ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + + def __init__(self, registry_info, vc_ver=None): + self.ri = registry_info + self.pi = self.ri.pi + self.vc_ver = vc_ver or self._find_latest_available_vc_ver() + + def _find_latest_available_vc_ver(self): + try: + return self.find_available_vc_vers()[-1] + except IndexError: + err = 'No Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + def find_available_vc_vers(self): + """ + Find all available Microsoft Visual C++ versions. + """ + ms = self.ri.microsoft + vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) + vc_vers = [] + for hkey in self.ri.HKEYS: + for key in vckeys: + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + try: + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + for i in range(subkeys): + try: + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + return sorted(vc_vers) + + @property + def VSInstallDir(self): + """ + Microsoft Visual Studio directory. + """ + # Default path + name = 'Microsoft Visual Studio %0.1f' % self.vc_ver + default = os.path.join(self.ProgramFilesx86, name) + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + + @property + def VCInstallDir(self): + """ + Microsoft Visual C++ directory. + """ + self.VSInstallDir + + guess_vc = self._guess_vc() or self._guess_vc_legacy() + + # Try to get "VC++ for Python" path from registry as default path + reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc + + # Try to get path from registry, if fail use default path + path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + + if not os.path.isdir(path): + msg = 'Microsoft Visual C++ directory not found' + raise distutils.errors.DistutilsPlatformError(msg) + + return path + + def _guess_vc(self): + """ + Locate Visual C for 2017 + """ + if self.vc_ver <= 14.0: + return + + default = r'VC\Tools\MSVC' + guess_vc = os.path.join(self.VSInstallDir, default) + # Subdir with VC exact version as name + try: + vc_exact_ver = os.listdir(guess_vc)[-1] + return os.path.join(guess_vc, vc_exact_ver) + except (OSError, IOError, IndexError): + pass + + def _guess_vc_legacy(self): + """ + Locate Visual C for versions prior to 2017 + """ + default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver + return os.path.join(self.ProgramFilesx86, default) + + @property + def WindowsSdkVersion(self): + """ + Microsoft Windows SDK versions for specified MSVC++ version. + """ + if self.vc_ver <= 9.0: + return ('7.0', '6.1', '6.0a') + elif self.vc_ver == 10.0: + return ('7.1', '7.0a') + elif self.vc_ver == 11.0: + return ('8.0', '8.0a') + elif self.vc_ver == 12.0: + return ('8.1', '8.1a') + elif self.vc_ver >= 14.0: + return ('10.0', '8.1') + + @property + def WindowsSdkLastVersion(self): + """ + Microsoft Windows SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.WindowsSdkDir, 'lib')) + + @property + def WindowsSdkDir(self): + """ + Microsoft Windows SDK directory. + """ + sdkdir = '' + for ver in self.WindowsSdkVersion: + # Try to get it from registry + loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + sdkdir = self.ri.lookup(loc, 'installationfolder') + if sdkdir: + break + if not sdkdir or not os.path.isdir(sdkdir): + # Try to get "VC++ for Python" version from registry + path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + install_base = self.ri.lookup(path, 'installdir') + if install_base: + sdkdir = os.path.join(install_base, 'WinSDK') + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default new path + for ver in self.WindowsSdkVersion: + intver = ver[:ver.rfind('.')] + path = r'Microsoft SDKs\Windows Kits\%s' % (intver) + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default old path + for ver in self.WindowsSdkVersion: + path = r'Microsoft SDKs\Windows\v%s' % ver + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir: + # If fail, use Platform SDK + sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + return sdkdir + + @property + def WindowsSDKExecutablePath(self): + """ + Microsoft Windows SDK executable directory. + """ + # Find WinSDK NetFx Tools registry dir name + if self.vc_ver <= 11.0: + netfxver = 35 + arch = '' + else: + netfxver = 40 + hidex86 = True if self.vc_ver <= 12.0 else False + arch = self.pi.current_dir(x64=True, hidex86=hidex86) + fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) + + # liste all possibles registry paths + regpaths = [] + if self.vc_ver >= 14.0: + for ver in self.NetFxSdkVersion: + regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + + for ver in self.WindowsSdkVersion: + regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + + # Return installation folder from the more recent path + for path in regpaths: + execpath = self.ri.lookup(path, 'installationfolder') + if execpath: + break + return execpath + + @property + def FSharpInstallDir(self): + """ + Microsoft Visual F# directory. + """ + path = r'%0.1f\Setup\F#' % self.vc_ver + path = os.path.join(self.ri.visualstudio, path) + return self.ri.lookup(path, 'productdir') or '' + + @property + def UniversalCRTSdkDir(self): + """ + Microsoft Universal CRT SDK directory. + """ + # Set Kit Roots versions for specified MSVC++ version + if self.vc_ver >= 14.0: + vers = ('10', '81') + else: + vers = () + + # Find path of the more recent Kit + for ver in vers: From pypy.commits at gmail.com Mon Feb 12 14:27:14 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 11:27:14 -0800 (PST) Subject: [pypy-commit] pypy py3.5: show no preference for compiler version used to compile host python Message-ID: <5a81ea92.15741c0a.432eb.d4f1@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93815:0526dd47e0cb Date: 2018-02-12 14:25 -0500 http://bitbucket.org/pypy/pypy/changeset/0526dd47e0cb/ Log: show no preference for compiler version used to compile host python diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -29,8 +29,8 @@ return None def Windows(cc=None, ver0=None): - if ver0 is None: - ver0 = _get_vcver0() + #if ver0 is None: + # ver0 = _get_vcver0() return _get_compiler_type(cc, False, ver0=ver0) def Windows_x64(cc=None, ver0=None): From pypy.commits at gmail.com Mon Feb 12 22:24:48 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 19:24:48 -0800 (PST) Subject: [pypy-commit] pypy default: raise IOError not OSError Message-ID: <5a825a80.c3c51c0a.99c6b.26bb@mx.google.com> Author: Matti Picus Branch: Changeset: r93816:fc85b98a24e2 Date: 2018-02-12 21:45 -0500 http://bitbucket.org/pypy/pypy/changeset/fc85b98a24e2/ Log: raise IOError not OSError diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -91,7 +91,10 @@ """fdopen(fd [, mode='r' [, buffering]]) -> file_object Return an open file object connected to a file descriptor.""" - posix.fstat(fd) + try: + posix.fstat(fd) + except OSError as e: + raise IOError(e.errno, e.message) return _fdopen(fd, mode, buffering) else: From pypy.commits at gmail.com Mon Feb 12 22:24:50 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 19:24:50 -0800 (PST) Subject: [pypy-commit] pypy py3.5: remove traces of is_valid_fd since merge of msvc14 Message-ID: <5a825a82.d4951c0a.7c18e.be63@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93817:46506daf1ecd Date: 2018-02-12 22:23 -0500 http://bitbucket.org/pypy/pypy/changeset/46506daf1ecd/ Log: remove traces of is_valid_fd since merge of msvc14 diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -93,10 +93,6 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) -# Capture file.fdopen at import time, as some code replaces -# __builtins__.file with a custom function. -_fdopen = file.fdopen - class uname_result(metaclass=structseqtype): @@ -117,14 +113,6 @@ columns = structseqfield(0, "width of the terminal window in characters") lines = structseqfield(1, "height of the terminal window in characters") -if osname == 'posix': - # POSIX: we want to check the file descriptor when fdopen() is called, - # not later when we read or write data. So we call fstat(), letting - # it raise if fd is invalid. - _validate_fd = posix.fstat -else: - _validate_fd = validate_fd - class times_result(metaclass=structseqtype): diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -2223,8 +2223,9 @@ Return a string describing the encoding of the device if the output is a terminal; else return None. """ - if not (rposix.is_valid_fd(fd) and os.isatty(fd)): - return space.w_None + with rposix.FdValidator(fd): + if not (os.isatty(fd)): + return space.w_None if _WIN32: if fd == 0: ccp = rwin32.GetConsoleCP() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -259,8 +259,6 @@ "signal.set_wakeup_fd is not implemented on Windows") if fd != -1: - if not rposix.is_valid_fd(fd): - raise oefmt(space.w_ValueError, "invalid fd") try: os.fstat(fd) flags = rposix.get_status_flags(fd) From pypy.commits at gmail.com Mon Feb 12 22:58:03 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Feb 2018 19:58:03 -0800 (PST) Subject: [pypy-commit] buildbot default: unschedule BSD nightly builds Message-ID: <5a82624b.50b91c0a.cbfc7.2bcd@mx.google.com> Author: Matti Picus Branch: Changeset: r1058:c4060955142b Date: 2018-02-12 22:57 -0500 http://bitbucket.org/pypy/buildbot/changeset/c4060955142b/ Log: unschedule BSD nightly builds diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -296,7 +296,7 @@ JITWIN32, # on allegro_win32, SalsaSalsa #JITFREEBSD764, # on headless #JITFREEBSD864, # on ananke - JITFREEBSD964, # on tavendo + #JITFREEBSD964, # on tavendo JITMACOSX64, # on xerxes # buildbot selftest #PYPYBUILDBOT # on cobra @@ -383,7 +383,7 @@ JITLINUX64, JITMACOSX64, JITWIN32, - JITFREEBSD964, + #JITFREEBSD964, JITONLYLINUXPPC64, JITBENCH, From pypy.commits at gmail.com Tue Feb 13 08:22:11 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Feb 2018 05:22:11 -0800 (PST) Subject: [pypy-commit] pypy default: disable mapdict-size-limit, broken for unicode Message-ID: <5a82e683.0ec61c0a.716e3.7087@mx.google.com> Author: Matti Picus Branch: Changeset: r93818:8fdeeb9d6730 Date: 2018-02-13 08:17 -0500 http://bitbucket.org/pypy/pypy/changeset/8fdeeb9d6730/ Log: disable mapdict-size-limit, broken for unicode diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -27,7 +27,7 @@ # the maximum number of attributes stored in mapdict (afterwards just use a # dict) -LIMIT_MAP_ATTRIBUTES = 80 +LIMIT_MAP_ATTRIBUTES = 8000 class AbstractAttribute(object): From pypy.commits at gmail.com Tue Feb 13 08:22:14 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Feb 2018 05:22:14 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5, disable mapdict-size-limit Message-ID: <5a82e686.8583df0a.e7cbe.4580@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93819:38f2eb1313e4 Date: 2018-02-13 08:19 -0500 http://bitbucket.org/pypy/pypy/changeset/38f2eb1313e4/ Log: merge default into py3.5, disable mapdict-size-limit diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -27,7 +27,7 @@ # the maximum number of attributes stored in mapdict (afterwards just use a # dict) -LIMIT_MAP_ATTRIBUTES = 80 +LIMIT_MAP_ATTRIBUTES = 8000 class AbstractAttribute(object): From pypy.commits at gmail.com Tue Feb 13 15:35:22 2018 From: pypy.commits at gmail.com (Alexander Schremmer) Date: Tue, 13 Feb 2018 12:35:22 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Add myself to winter sprint. Message-ID: <5a834c0a.46061c0a.b4ecc.bf8a@mx.google.com> Author: Alexander Schremmer Branch: extradoc Changeset: r5868:12294fb40176 Date: 2018-02-13 21:32 +0100 http://bitbucket.org/pypy/extradoc/changeset/12294fb40176/ Log: Add myself to winter sprint. diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -18,6 +18,7 @@ Arianna 17.3/21.3 own booking Ermina Floris Bruynooghe 17.3/21.3 Ermina Antonio Cuni 18.3/23.3 Ermina, exact dates to be confirmed +Alexander Schremmer 18.3/20.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Wed Feb 14 10:38:17 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 14 Feb 2018 07:38:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix mapdict-size-limit on pypy3 Message-ID: <5a8457e9.04191c0a.90d1.4e67@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r93820:ff6a031587c2 Date: 2018-02-14 16:37 +0100 http://bitbucket.org/pypy/pypy/changeset/ff6a031587c2/ Log: fix mapdict-size-limit on pypy3 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -4,6 +4,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.unicodehelper import encode_utf8 from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, @@ -27,7 +28,7 @@ # the maximum number of attributes stored in mapdict (afterwards just use a # dict) -LIMIT_MAP_ATTRIBUTES = 8000 +LIMIT_MAP_ATTRIBUTES = 80 class AbstractAttribute(object): @@ -431,7 +432,8 @@ def materialize_str_dict(self, space, obj, str_dict): new_obj = self.back.materialize_str_dict(space, obj, str_dict) if self.index == DICT: - str_dict[self.name] = obj._mapdict_read_storage(self.storageindex) + enc_name = encode_utf8(space, self.name) + str_dict[enc_name] = obj._mapdict_read_storage(self.storageindex) else: self._copy_attr(obj, new_obj) return new_obj @@ -767,7 +769,7 @@ def switch_to_text_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) - strategy = self.space.fromcache(BytesDictStrategy) + strategy = self.space.fromcache(UnicodeDictStrategy) str_dict = strategy.unerase(strategy.get_empty_storage()) w_dict.set_strategy(strategy) w_dict.dstorage = strategy.erase(str_dict) diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -125,6 +125,7 @@ obj.setdictvalue(space, str(i), i) # moved to dict (which is the remaining non-slot item) assert len(obj.storage) == 1 + numslots + assert isinstance(obj.getdict(space).dstrategy, UnicodeDictStrategy) for i in range(1000): assert obj.getdictvalue(space, str(i)) == i From pypy.commits at gmail.com Wed Feb 14 10:42:07 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 14 Feb 2018 07:42:07 -0800 (PST) Subject: [pypy-commit] pypy default: re-enable on default as well Message-ID: <5a8458cf.04191c0a.90d1.4f1c@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93821:87ece4f2df4d Date: 2018-02-14 16:41 +0100 http://bitbucket.org/pypy/pypy/changeset/87ece4f2df4d/ Log: re-enable on default as well diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -27,7 +27,7 @@ # the maximum number of attributes stored in mapdict (afterwards just use a # dict) -LIMIT_MAP_ATTRIBUTES = 8000 +LIMIT_MAP_ATTRIBUTES = 80 class AbstractAttribute(object): From pypy.commits at gmail.com Thu Feb 15 03:41:08 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 15 Feb 2018 00:41:08 -0800 (PST) Subject: [pypy-commit] cffi default: Trying to fix ffi.dlopen() for unicode filenames on Windows Message-ID: <5a8547a4.e9b2df0a.1fe2b.2844@mx.google.com> Author: Armin Rigo Branch: Changeset: r3096:93e213825746 Date: 2018-02-15 09:21 +0100 http://bitbucket.org/cffi/cffi/changeset/93e213825746/ Log: Trying to fix ffi.dlopen() for unicode filenames on Windows diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3989,7 +3989,7 @@ static PyObject *dl_repr(DynLibObject *dlobj) { - return PyText_FromFormat("", dlobj->dl_name); + return PyText_FromFormat("", dlobj->dl_name); } static PyObject *dl_load_function(DynLibObject *dlobj, PyObject *args) @@ -4013,7 +4013,7 @@ if (funcptr == NULL) { const char *error = dlerror(); PyErr_Format(PyExc_AttributeError, - "function/symbol '%s' not found in library '%s': %s", + "function/symbol '%s' not found in library %s: %s", funcname, dlobj->dl_name, error); return NULL; } @@ -4040,7 +4040,7 @@ const char *error = dlerror(); if (error != NULL) { PyErr_Format(PyExc_KeyError, - "variable '%s' not found in library '%s': %s", + "variable '%s' not found in library %s: %s", varname, dlobj->dl_name, error); return NULL; } @@ -4064,7 +4064,7 @@ if (data == NULL) { const char *error = dlerror(); PyErr_Format(PyExc_KeyError, - "variable '%s' not found in library '%s': %s", + "variable '%s' not found in library %s: %s", varname, dlobj->dl_name, error); return NULL; } @@ -4116,8 +4116,9 @@ static PyObject *b_load_library(PyObject *self, PyObject *args) { char *filename_or_null, *printable_filename; + PyObject *s = NULL; void *handle; - DynLibObject *dlobj; + DynLibObject *dlobj = NULL; int flags = 0; if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { @@ -4126,31 +4127,61 @@ &dummy, &flags)) return NULL; filename_or_null = NULL; - } - else if (!PyArg_ParseTuple(args, "et|i:load_library", - Py_FileSystemDefaultEncoding, &filename_or_null, - &flags)) - return NULL; - + printable_filename = ""; + } + else + { + printable_filename = NULL; + s = PyObject_Repr(PyTuple_GET_ITEM(args, 0)); + if (s != NULL) { + printable_filename = PyText_AsUTF8(s); + } + if (printable_filename == NULL) { + PyErr_Clear(); + printable_filename = "?"; + } + +#ifdef MS_WIN32 + { + Py_UNICODE *filenameW; + if (PyArg_ParseTuple(args, "u|i:load_library", &filenameW, &flags)) + { + handle = dlopenW(filenameW); + goto got_handle; + } + PyErr_Clear(); + } +#endif + if (!PyArg_ParseTuple(args, "et|i:load_library", + Py_FileSystemDefaultEncoding, &filename_or_null, + &flags)) + goto error; + } if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) flags |= RTLD_NOW; - printable_filename = filename_or_null ? filename_or_null : ""; handle = dlopen(filename_or_null, flags); + +#ifdef MS_WIN32 + got_handle: +#endif if (handle == NULL) { const char *error = dlerror(); PyErr_Format(PyExc_OSError, "cannot load library %s: %s", printable_filename, error); - return NULL; + goto error; } dlobj = PyObject_New(DynLibObject, &dl_type); if (dlobj == NULL) { dlclose(handle); - return NULL; + goto error; } dlobj->dl_handle = handle; dlobj->dl_name = strdup(printable_filename); + + error: + Py_XDECREF(s); return (PyObject *)dlobj; } diff --git a/c/misc_win32.h b/c/misc_win32.h --- a/c/misc_win32.h +++ b/c/misc_win32.h @@ -192,7 +192,12 @@ static void *dlopen(const char *filename, int flag) { - return (void *)LoadLibrary(filename); + return (void *)LoadLibraryA(filename); +} + +static void *dlopenW(const wchar_t *filename) +{ + return (void *)LoadLibraryW(filename); } static void *dlsym(void *handle, const char *symbol) diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py --- a/testing/cffi0/test_ownlib.py +++ b/testing/cffi0/test_ownlib.py @@ -114,8 +114,12 @@ if sys.platform == 'win32': import os # did we already build it? - if os.path.exists(str(udir.join('testownlib.dll'))): - cls.module = str(udir.join('testownlib.dll')) + if cls.Backend is CTypesBackend: + dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend + else: + dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') # non-ascii char + if os.path.exists(dll_path): + cls.module = dll_path return # try (not too hard) to find the version used to compile this python # no mingw @@ -135,8 +139,9 @@ if os.path.isfile(vcvarsall): cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ ' /LD /Fetestownlib.dll' - subprocess.check_call(cmd, cwd = str(udir), shell=True) - cls.module = str(udir.join('testownlib.dll')) + subprocess.check_call(cmd, cwd = str(udir), shell=True) + os.rename(str(udir) + '\\testownlib.dll', dll_path) + cls.module = dll_path else: subprocess.check_call( 'cc testownlib.c -shared -fPIC -o testownlib.so', From pypy.commits at gmail.com Thu Feb 15 07:54:56 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 15 Feb 2018 04:54:56 -0800 (PST) Subject: [pypy-commit] pypy winapi: merge py3.5 into branch Message-ID: <5a858320.d32f1c0a.6005.4191@mx.google.com> Author: Matti Picus Branch: winapi Changeset: r93822:bc815f3e455d Date: 2018-02-15 14:53 +0200 http://bitbucket.org/pypy/pypy/changeset/bc815f3e455d/ Log: merge py3.5 into branch diff too long, truncating to 2000 out of 5761 lines diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py --- a/lib-python/3/distutils/unixccompiler.py +++ b/lib-python/3/distutils/unixccompiler.py @@ -222,6 +222,10 @@ return "-L" + dir def _is_gcc(self, compiler_name): + if "__pypy__" in sys.builtin_module_names: # issue #2747 + if (compiler_name.startswith('cc') or + compiler_name.startswith('c++')): + return True return "gcc" in compiler_name or "g++" in compiler_name def runtime_library_dir_option(self, dir): diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -155,9 +155,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements, uri) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements, uri) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.3 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,37 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -2,7 +2,7 @@ import __pypy__ import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,33 @@ .. branch: cpyext-datetime2 Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1408,3 +1408,11 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -207,6 +207,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,6 +1,7 @@ import py import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be) from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp @@ -51,7 +52,10 @@ py.test.raises(Hit, decode_utf8, space, "\xed\xb0\x80") py.test.raises(Hit, decode_utf8, space, "\xed\xa0\x80\xed\xb0\x80") got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] def test_decode_utf8_allow_surrogates(): sp = FakeSpace() diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -105,14 +105,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ @@ -134,7 +126,7 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.3" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -589,12 +589,17 @@ assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys import codecs sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' assert b'\x00\x60\x80'.decode( diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py --- a/pypy/module/_multiprocessing/__init__.py +++ b/pypy/module/_multiprocessing/__init__.py @@ -16,4 +16,4 @@ interpleveldefs['recv'] = 'interp_win32_py3.multiprocessing_recv' interpleveldefs['send'] = 'interp_win32_py3.multiprocessing_send' else: - interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink', + interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink' diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1202,12 +1202,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -300,6 +300,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -31,7 +31,7 @@ from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -97,25 +97,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h --- a/pypy/module/cpyext/include/longobject.h +++ b/pypy/module/cpyext/include/longobject.h @@ -20,6 +20,9 @@ #define PyLong_AS_LONG(op) PyLong_AsLong(op) +#define _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) \ + _PyLong_AsByteArrayO((PyObject *)(v), bytes, n, little_endian, is_signed) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -3,8 +3,8 @@ cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t, CONST_STRING, ADDR, CANNOT_FAIL) from pypy.objspace.std.longobject import W_LongObject -from pypy.interpreter.error import OperationError -from rpython.rlib.rbigint import rbigint +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rbigint import rbigint, InvalidSignednessError PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long") @@ -251,3 +251,26 @@ byteorder = 'big' result = rbigint.frombytes(s, byteorder, signed != 0) return space.newlong_from_rbigint(result) + + at cpython_api([PyObject, rffi.UCHARP, rffi.SIZE_T, + rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1) +def _PyLong_AsByteArrayO(space, w_v, bytes, n, little_endian, is_signed): + n = rffi.cast(lltype.Signed, n) + little_endian = rffi.cast(lltype.Signed, little_endian) + signed = rffi.cast(lltype.Signed, is_signed) != 0 + byteorder = 'little' if little_endian else 'big' + bigint = space.bigint_w(w_v) + try: + digits = bigint.tobytes(n, byteorder, signed) + except InvalidSignednessError: # < 0 but not 'signed' + # in this case, CPython raises OverflowError even though the C + # comments say it should raise TypeError + raise oefmt(space.w_OverflowError, + "can't convert negative long to unsigned") + except OverflowError: + raise oefmt(space.w_OverflowError, + "long too big to convert") + assert len(digits) == n + for i in range(n): + bytes[i] = rffi.cast(rffi.UCHAR, digits[i]) + return 0 diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject +from pypy.objspace.std.boolobject import W_BoolObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here @@ -21,6 +23,52 @@ #________________________________________________________ # type description +class W_BaseCPyObject(W_ObjectObject): + """ A subclass of W_ObjectObject that has one field for directly storing + the link from the w_obj to the cpy ref. This is only used for C-defined + types. """ + + +def check_true(s_arg, bookeeper): + assert s_arg.const is True + +def w_root_as_pyobj(w_obj, space): + from rpython.rlib.debug import check_annotation + # make sure that translation crashes if we see this while not translating + # with cpyext + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + # default implementation of _cpyext_as_pyobj + return rawrefcount.from_obj(PyObject, w_obj) + +def w_root_attach_pyobj(w_obj, space, py_obj): + from rpython.rlib.debug import check_annotation + check_annotation(space.config.objspace.usemodules.cpyext, check_true) + assert space.config.objspace.usemodules.cpyext + # default implementation of _cpyext_attach_pyobj + rawrefcount.create_link_pypy(w_obj, py_obj) + + +def add_direct_pyobj_storage(cls): + """ Add the necessary methods to a class to store a reference to the py_obj + on its instances directly. """ + + cls._cpy_ref = lltype.nullptr(PyObject.TO) + + def _cpyext_as_pyobj(self, space): + return self._cpy_ref + cls._cpyext_as_pyobj = _cpyext_as_pyobj + + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + cls._cpyext_attach_pyobj = _cpyext_attach_pyobj + +add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_TypeObject) +add_direct_pyobj_storage(W_NoneObject) +add_direct_pyobj_storage(W_BoolObject) + + class BaseCpyTypedescr(object): basestruct = PyObject.TO W_BaseObject = W_ObjectObject @@ -66,8 +114,12 @@ def realize(self, space, obj): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) + assert isinstance(w_type, W_TypeObject) try: - w_obj = space.allocate_instance(self.W_BaseObject, w_type) + if w_type.flag_cpytype: + w_obj = space.allocate_instance(W_BaseCPyObject, w_type) + else: + w_obj = space.allocate_instance(self.W_BaseObject, w_type) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_SystemError, @@ -76,6 +128,9 @@ w_type) raise track_reference(space, obj, w_obj) + if w_type.flag_cpytype: + assert isinstance(w_obj, W_BaseCPyObject) + w_obj._cpy_ref = obj return w_obj typedescr_cache = {} @@ -186,12 +241,12 @@ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. The reference in 'py_obj' is not stolen! Remember to decref() - it is you need to. + it if you need to. """ # XXX looks like a PyObject_GC_TRACK assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY - rawrefcount.create_link_pypy(w_obj, py_obj) + w_obj._cpyext_attach_pyobj(space, py_obj) w_marker_deallocating = W_Root() @@ -237,7 +292,7 @@ @jit.dont_look_inside def as_pyobj(space, w_obj, w_userdata=None, immortal=False): """ - Returns a 'PyObject *' representing the given intepreter object. + Returns a 'PyObject *' representing the given interpreter object. This doesn't give a new reference, but the returned 'PyObject *' is valid at least as long as 'w_obj' is. **To be safe, you should use keepalive_until_here(w_obj) some time later.** In case of @@ -245,7 +300,7 @@ """ assert not is_pyobj(w_obj) if w_obj is not None: - py_obj = rawrefcount.from_obj(PyObject, w_obj) + py_obj = w_obj._cpyext_as_pyobj(space) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) # diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -267,7 +267,7 @@ raise oefmt(space.w_ValueError, "sequence.index(x): x not in sequence") class CPyListStrategy(ListStrategy): - erase, unerase = rerased.new_erasing_pair("empty") + erase, unerase = rerased.new_erasing_pair("cpylist") erase = staticmethod(erase) unerase = staticmethod(unerase) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1577,13 +1577,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF32String(space, unicode): - """Return a Python byte string using the UTF-32 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP, Py_ssize_t], PyObject) def PyUnicode_DecodeUTF16Stateful(space, s, size, errors, byteorder, consumed): """If consumed is NULL, behave like PyUnicode_DecodeUTF16(). If @@ -1612,13 +1605,6 @@ Return NULL if an exception was raised by the codec.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyUnicode_AsUTF16String(space, unicode): - """Return a Python byte string using the UTF-16 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return NULL if an exception was raised by the codec.""" - raise NotImplementedError - @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP], PyObject) def PyUnicode_DecodeUTF7(space, s, size, errors): """Create a Unicode object by decoding size bytes of the UTF-7 encoded string diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -259,6 +259,48 @@ assert module.from_bytearray(False, False) == 0x9ABC41 assert module.from_bytearray(False, True) == -0x6543BF + def test_asbytearray(self): + module = self.import_extension('foo', [ + ("as_bytearray", "METH_VARARGS", + """ + PyObject *result; + PyLongObject *o; + int n, little_endian, is_signed; + unsigned char *bytes; + if (!PyArg_ParseTuple(args, "O!iii", &PyLong_Type, &o, &n, + &little_endian, &is_signed)) + return NULL; + bytes = malloc(n); + if (_PyLong_AsByteArray(o, bytes, (size_t)n, + little_endian, is_signed) != 0) + { + free(bytes); + return NULL; + } + result = PyString_FromStringAndSize((const char *)bytes, n); + free(bytes); + return result; + """), + ]) + s = module.as_bytearray(0x41BC9A, 4, True, False) + assert s == "\x9A\xBC\x41\x00" + s = module.as_bytearray(0x41BC9A, 4, False, False) + assert s == "\x00\x41\xBC\x9A" + s = module.as_bytearray(0x41BC9A, 3, True, False) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x41BC9A, 3, True, True) + assert s == "\x9A\xBC\x41" + s = module.as_bytearray(0x9876, 2, True, False) + assert s == "\x76\x98" + s = module.as_bytearray(0x9876 - 0x10000, 2, True, True) + assert s == "\x76\x98" + raises(OverflowError, module.as_bytearray, + 0x9876, 2, False, True) + raises(OverflowError, module.as_bytearray, + -1, 2, True, False) + raises(OverflowError, module.as_bytearray, + 0x1234567, 3, True, False) + def test_fromunicode(self): module = self.import_extension('foo', [ ("from_unicode", "METH_O", diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -218,7 +218,7 @@ if not cls.runappdirect: cls.total_mem = 0 - def add_memory_pressure(estimate): + def add_memory_pressure(estimate, object=None): assert estimate >= 0 cls.total_mem += estimate cls.orig_add_memory_pressure = [rgc.add_memory_pressure] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -3,13 +3,23 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call -from pypy.module.cpyext.pyobject import make_ref, from_ref, decref +from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr import sys import pytest class AppTestTypeObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.im_func(cls) + def _check_uses_shortcut(w_inst): + res = hasattr(w_inst, "_cpy_ref") and w_inst._cpy_ref + res = res and as_pyobj(cls.space, w_inst) == w_inst._cpy_ref + return cls.space.newbool(res) + cls.w__check_uses_shortcut = cls.space.wrap( + gateway.interp2app(_check_uses_shortcut)) + def test_typeobject(self): import sys module = self.import_module(name='foo') @@ -162,6 +172,25 @@ assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) + def test_shortcut(self): + # test that instances of classes that are defined in C become an + # instance of W_BaseCPyObject and thus can be converted faster back to + # their pyobj, because they store a pointer to it directly. + if self.runappdirect: + skip("can't run with -A") + module = self.import_module(name='foo') + obj = module.fooType() + assert self._check_uses_shortcut(obj) + # W_TypeObjects use shortcut + assert self._check_uses_shortcut(object) + assert self._check_uses_shortcut(type) + # None, True, False use shortcut + assert self._check_uses_shortcut(None) + assert self._check_uses_shortcut(True) + assert self._check_uses_shortcut(False) + assert not self._check_uses_shortcut(1) + assert not self._check_uses_shortcut(object()) + def test_multiple_inheritance1(self): module = self.import_module(name='foo') obj = module.UnicodeSubtype(u'xyz') diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -359,6 +359,20 @@ m = self.import_module('_widechar') raises(ValueError, m.test_widechar) + def test_AsUTFNString(self): + module = self.import_extension('foo', [ + ("asutf8", "METH_O", "return PyUnicode_AsUTF8String(args);"), + ("asutf16", "METH_O", "return PyUnicode_AsUTF16String(args);"), + ("asutf32", "METH_O", "return PyUnicode_AsUTF32String(args);"), + ]) + u = u'sp\x09m\u1234\U00012345' + s = module.asutf8(u) + assert s == u.encode('utf-8') + s = module.asutf16(u) + assert s == u.encode('utf-16') + s = module.asutf32(u) + assert s == u.encode('utf-32') + class TestUnicode(BaseApiTest): def test_unicodeobject(self, space): @@ -448,10 +462,24 @@ lltype.free(ar, flavor='raw') def test_AsUTF8String(self, space): - w_u = space.wrap(u'sp\x09m') + w_u = space.wrap(u'sp\x09m\u1234') w_res = PyUnicode_AsUTF8String(space, w_u) assert space.type(w_res) is space.w_bytes - assert space.unwrap(w_res) == 'sp\tm' + assert space.unwrap(w_res) == 'sp\tm\xe1\x88\xb4' + + def test_AsUTF16String(self, space): + u = u'sp\x09m\u1234\U00012345' + w_u = space.wrap(u) + w_res = PyUnicode_AsUTF16String(space, w_u) + assert space.type(w_res) is space.w_bytes + assert space.unwrap(w_res) == u.encode('utf-16') + + def test_AsUTF32String(self, space): + u = u'sp\x09m\u1234\U00012345' + w_u = space.wrap(u) + w_res = PyUnicode_AsUTF32String(space, w_u) + assert space.type(w_res) is space.w_bytes + assert space.unwrap(w_res) == u.encode('utf-32') def test_decode_utf8(self, space): u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -341,8 +341,12 @@ if len(slot_names) == 1: func = getattr(pto, slot_names[0]) if slot_names[0] == 'c_tp_hash': - if hash_not_impl == func: - # special case for tp_hash == PyObject_HashNotImplemented + # two special cases where __hash__ is explicitly set to None + # (which leads to an unhashable type): + # 1) tp_hash == PyObject_HashNotImplemented + # 2) tp_hash == NULL and tp_richcompare not NULL + if hash_not_impl == func or ( + not func and pto.c_tp_richcompare): dict_w[method_name] = space.w_None continue else: diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -716,7 +716,7 @@ ref[0] = rffi.cast(PyObject, py_newuni) return 0 -def make_conversion_functions(suffix, encoding): +def make_conversion_functions(suffix, encoding, only_for_asstring=False): @cpython_api([PyObject], PyObject) @func_renamer('PyUnicode_As%sString' % suffix) def PyUnicode_AsXXXString(space, w_unicode): @@ -728,6 +728,9 @@ return unicodeobject.encode_object(space, w_unicode, encoding, "strict") globals()['PyUnicode_As%sString' % suffix] = PyUnicode_AsXXXString + if only_for_asstring: + return + @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING], PyObject) @func_renamer('PyUnicode_Decode%s' % suffix) def PyUnicode_DecodeXXX(space, s, size, errors): @@ -758,6 +761,8 @@ globals()['PyUnicode_Encode%s' % suffix] = PyUnicode_EncodeXXX make_conversion_functions('UTF8', 'utf-8') +make_conversion_functions('UTF16', 'utf-16', only_for_asstring=True) +make_conversion_functions('UTF32', 'utf-32', only_for_asstring=True) make_conversion_functions('ASCII', 'ascii') make_conversion_functions('Latin1', 'latin-1') if sys.platform == 'win32': diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -19,6 +19,7 @@ space.config.translation.gctransformer == "framework"): self.appleveldefs.update({ 'dump_rpy_heap': 'app_referents.dump_rpy_heap', + 'get_stats': 'app_referents.get_stats', }) self.interpleveldefs.update({ 'get_rpy_roots': 'referents.get_rpy_roots', @@ -28,6 +29,7 @@ 'get_objects': 'referents.get_objects', 'get_referents': 'referents.get_referents', 'get_referrers': 'referents.get_referrers', + '_get_stats': 'referents.get_stats', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -48,3 +48,66 @@ file.flush() fd = file.fileno() gc._dump_rpy_heap(fd) + +class GcStats(object): + def __init__(self, s): + self._s = s + for item in ('total_gc_memory', 'jit_backend_used', + 'total_memory_pressure', + 'total_allocated_memory', 'jit_backend_allocated', + 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', + 'total_rawmalloced_memory', 'nursery_size', + 'peak_arena_memory', 'peak_rawmalloced_memory'): + setattr(self, item, self._format(getattr(self._s, item))) + self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + + self._s.jit_backend_used) + self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + + self._s.jit_backend_allocated) + + def _format(self, v): + if v < 1000000: + # bit unlikely ;-) + return "%.1fkB" % (v / 1024.) + return "%.1fMB" % (v / 1024. / 1024.) + + def __repr__(self): + if self._s.total_memory_pressure != -1: + extra = "\nmemory pressure: %s" % self.total_memory_pressure + else: + extra = "" + return """Total memory consumed: + GC used: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler used: %s%s + ----------------------------- + Total: %s + + Total memory allocated: + GC allocated: %s (peak: %s) + in arenas: %s + rawmalloced: %s + nursery: %s + raw assembler allocated: %s%s + ----------------------------- + Total: %s + """ % (self.total_gc_memory, self.peak_memory, + self.total_arena_memory, + self.total_rawmalloced_memory, + self.nursery_size, + self.jit_backend_used, + extra, + self.memory_used_sum, + + self.total_allocated_memory, self.peak_allocated_memory, + self.peak_arena_memory, + self.peak_rawmalloced_memory, + self.nursery_size, + self.jit_backend_allocated, + extra, + self.memory_allocated_sum) + + +def get_stats(): + return GcStats(gc._get_stats()) diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -1,7 +1,7 @@ -from rpython.rlib import rgc +from rpython.rlib import rgc, jit_hooks from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.error import oefmt, wrap_oserror from rpython.rlib.objectmodel import we_are_translated @@ -170,3 +170,53 @@ l = rgc.get_typeids_list() list_w = [space.newint(l[i]) for i in range(len(l))] return space.newlist(list_w) + +class W_GcStats(W_Root): + def __init__(self, memory_pressure): + if memory_pressure: + self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE) + else: + self.total_memory_pressure = -1 + self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY) + self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY) + self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY) + self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY) + self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None) + self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None) + self.total_arena_memory = rgc.get_stats(rgc.TOTAL_ARENA_MEMORY) + self.total_rawmalloced_memory = rgc.get_stats( + rgc.TOTAL_RAWMALLOCED_MEMORY) + self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) + self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) + self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + +W_GcStats.typedef = TypeDef("GcStats", + total_memory_pressure=interp_attrproperty("total_memory_pressure", + cls=W_GcStats, wrapfn="newint"), + total_gc_memory=interp_attrproperty("total_gc_memory", + cls=W_GcStats, wrapfn="newint"), + peak_allocated_memory=interp_attrproperty("peak_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + peak_memory=interp_attrproperty("peak_memory", + cls=W_GcStats, wrapfn="newint"), + total_allocated_memory=interp_attrproperty("total_allocated_memory", + cls=W_GcStats, wrapfn="newint"), + jit_backend_allocated=interp_attrproperty("jit_backend_allocated", + cls=W_GcStats, wrapfn="newint"), + jit_backend_used=interp_attrproperty("jit_backend_used", + cls=W_GcStats, wrapfn="newint"), + total_arena_memory=interp_attrproperty("total_arena_memory", + cls=W_GcStats, wrapfn="newint"), + total_rawmalloced_memory=interp_attrproperty("total_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + peak_arena_memory=interp_attrproperty("peak_arena_memory", + cls=W_GcStats, wrapfn="newint"), + peak_rawmalloced_memory=interp_attrproperty("peak_rawmalloced_memory", + cls=W_GcStats, wrapfn="newint"), + nursery_size=interp_attrproperty("nursery_size", + cls=W_GcStats, wrapfn="newint"), +) + + at unwrap_spec(memory_pressure=bool) +def get_stats(space, memory_pressure=False): + return W_GcStats(memory_pressure) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,6 +1,5 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import errno @@ -114,14 +113,6 @@ columns = structseqfield(0, "width of the terminal window in characters") lines = structseqfield(1, "height of the terminal window in characters") -if osname == 'posix': - # POSIX: we want to check the file descriptor when fdopen() is called, - # not later when we read or write data. So we call fstat(), letting - # it raise if fd is invalid. - _validate_fd = posix.fstat -else: - _validate_fd = validate_fd - class times_result(metaclass=structseqtype): diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -2223,8 +2223,9 @@ Return a string describing the encoding of the device if the output is a terminal; else return None. """ - if not (rposix.is_valid_fd(fd) and os.isatty(fd)): - return space.w_None + with rposix.FdValidator(fd): + if not (os.isatty(fd)): + return space.w_None if _WIN32: if fd == 0: ccp = rwin32.GetConsoleCP() diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -17,6 +17,7 @@ USEMODULES += ['fcntl', 'select', '_posixsubprocess', '_socket'] else: USEMODULES += ['_rawffi', 'thread'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -843,11 +843,11 @@ # Currently this is just the size of the pointer and some estimated bytes. # The struct isn't actually defined in expat.h - it is in xmlparse.c # XXX: find a good estimate of the XML_ParserStruct - rgc.add_memory_pressure(XML_Parser_SIZE + 300) if not xmlparser: raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed") parser = W_XMLParserType(space, xmlparser, w_intern) + rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser) XML_SetUnknownEncodingHandler( parser.itself, UnknownEncodingHandlerData_callback, rffi.cast(rffi.VOIDP, parser.id)) diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -259,8 +259,6 @@ "signal.set_wakeup_fd is not implemented on Windows") if fd != -1: - if not rposix.is_valid_fd(fd): - raise oefmt(space.w_ValueError, "invalid fd") try: os.fstat(fd) flags = rposix.get_status_flags(fd) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2298,3 +2298,11 @@ else: assert lib.__loader__ is None assert lib.__spec__ is None + +def test_realize_struct_error(): + ffi = FFI() + ffi.cdef("""typedef ... foo_t; struct foo_s { void (*x)(foo_t); };""") + lib = verify(ffi, "test_realize_struct_error", """ + typedef int foo_t; struct foo_s { void (*x)(foo_t); }; + """) + py.test.raises(TypeError, ffi.new, "struct foo_s *") diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -221,9 +221,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -332,20 +333,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { return _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -359,8 +374,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -4,10 +4,11 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.unicodehelper import encode_utf8 from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, - W_DictObject, + W_DictObject, BytesDictStrategy, UnicodeDictStrategy ) from pypy.objspace.std.typeobject import MutableCell @@ -25,6 +26,10 @@ # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative +# the maximum number of attributes stored in mapdict (afterwards just use a +# dict) +LIMIT_MAP_ATTRIBUTES = 80 + class AbstractAttribute(object): _immutable_fields_ = ['terminator'] @@ -253,6 +258,9 @@ def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") + def materialize_str_dict(self, space, obj, str_dict): + raise NotImplementedError("abstract base class") + def remove_dict_entries(self, obj): raise NotImplementedError("abstract base class") @@ -272,6 +280,13 @@ def _write_terminator(self, obj, name, index, w_value): obj._get_mapdict_map().add_attr(obj, name, index, w_value) + if index == DICT and obj._get_mapdict_map().length() >= LIMIT_MAP_ATTRIBUTES: + space = self.space + w_dict = obj.getdict(space) + assert isinstance(w_dict, W_DictMultiObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, MapDictStrategy) + strategy.switch_to_text_strategy(w_dict) return True def copy(self, obj): @@ -302,6 +317,12 @@ self.devolved_dict_terminator = DevolvedDictTerminator(space, w_cls) def materialize_r_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def materialize_str_dict(self, space, obj, dict_w): + return self._make_devolved(space) + + def _make_devolved(self, space): result = Object() result.space = space result._mapdict_init_empty(self.devolved_dict_terminator) @@ -408,6 +429,15 @@ self._copy_attr(obj, new_obj) return new_obj + def materialize_str_dict(self, space, obj, str_dict): + new_obj = self.back.materialize_str_dict(space, obj, str_dict) + if self.index == DICT: + enc_name = encode_utf8(space, self.name) + str_dict[enc_name] = obj._mapdict_read_storage(self.storageindex) + else: + self._copy_attr(obj, new_obj) + return new_obj + def remove_dict_entries(self, obj): new_obj = self.back.remove_dict_entries(obj) if self.index != DICT: @@ -737,6 +767,15 @@ assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) + def switch_to_text_strategy(self, w_dict): + w_obj = self.unerase(w_dict.dstorage) + strategy = self.space.fromcache(UnicodeDictStrategy) + str_dict = strategy.unerase(strategy.get_empty_storage()) + w_dict.set_strategy(strategy) + w_dict.dstorage = strategy.erase(str_dict) + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None + materialize_str_dict(self.space, w_obj, str_dict) + def getitem(self, w_dict, w_key): space = self.space w_lookup_type = space.type(w_key) @@ -832,6 +871,11 @@ new_obj = map.materialize_r_dict(space, obj, dict_w) obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) +def materialize_str_dict(space, obj, dict_w): + map = obj._get_mapdict_map() + new_obj = map.materialize_str_dict(space, obj, dict_w) + obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) + class IteratorMixin(object): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -114,6 +114,34 @@ assert obj2.getdictvalue(space, "b") == 60 assert obj2.map is obj.map +def test_add_attribute_limit(): + for numslots in [0, 10, 100]: + cls = Class() + obj = cls.instantiate() + for i in range(numslots): + obj.setslotvalue(i, i) # some extra slots too, sometimes + # test that eventually attributes are really just stored in a dictionary + for i in range(1000): + obj.setdictvalue(space, str(i), i) + # moved to dict (which is the remaining non-slot item) + assert len(obj.storage) == 1 + numslots + assert isinstance(obj.getdict(space).dstrategy, UnicodeDictStrategy) + + for i in range(1000): + assert obj.getdictvalue(space, str(i)) == i + for i in range(numslots): + assert obj.getslotvalue(i) == i # check extra slots + + # this doesn't happen with slots + cls = Class() + obj = cls.instantiate() + for i in range(1000): + obj.setslotvalue(i, i) + assert len(obj.storage) == 1000 + + for i in range(1000): + assert obj.getslotvalue(i) == i + def test_insert_different_orders(): cls = Class() obj = cls.instantiate() @@ -797,7 +825,6 @@ assert d == {} def test_change_class_slots(self): - skip("not supported by pypy yet") class A(object): __slots__ = ["x", "y"] @@ -815,7 +842,6 @@ assert isinstance(a, B) def test_change_class_slots_dict(self): - skip("not supported by pypy yet") class A(object): __slots__ = ["x", "__dict__"] class B(object): @@ -843,7 +869,7 @@ assert a.y == 2 d = a.__dict__ d[1] = 3 - assert d == {"x": 1, "y": 2, 1:3} + assert d == {"y": 2, 1: 3} a.__class__ = B assert a.x == 1 assert a.y == 2 diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -71,6 +71,7 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() + self.memory_pressure_types = set() self.register_builtins() diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py From pypy.commits at gmail.com Thu Feb 15 08:01:05 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 15 Feb 2018 05:01:05 -0800 (PST) Subject: [pypy-commit] pypy default: fixes Message-ID: <5a858491.85261c0a.19804.0589@mx.google.com> Author: fijal Branch: Changeset: r93823:df78a73afdf9 Date: 2018-02-15 14:00 +0100 http://bitbucket.org/pypy/pypy/changeset/df78a73afdf9/ Log: fixes diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,11 +1,12 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- .. _minimark-environment-variables: -Minimark --------- - PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -72,7 +72,7 @@ def __repr__(self): if self._s.total_memory_pressure != -1: - extra = "\nmemory pressure: %s" % self.total_memory_pressure + extra = "\n memory pressure: %s" % self.total_memory_pressure else: extra = "" return """Total memory consumed: @@ -109,5 +109,5 @@ self.memory_allocated_sum) -def get_stats(): - return GcStats(gc._get_stats()) +def get_stats(memory_pressure=False): + return GcStats(gc._get_stats(memory_pressure=memory_pressure)) From pypy.commits at gmail.com Thu Feb 15 08:06:23 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 15 Feb 2018 05:06:23 -0800 (PST) Subject: [pypy-commit] pypy default: write docs Message-ID: <5a8585cf.8f981c0a.934f2.997c@mx.google.com> Author: fijal Branch: Changeset: r93824:ed765f557481 Date: 2018-02-15 14:05 +0100 http://bitbucket.org/pypy/pypy/changeset/ed765f557481/ Log: write docs diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -5,8 +5,87 @@ Incminimark ----------- +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, without any effort, while surviving objects from the nursery +are copied to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're big +enough. + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a high memory fragmentation, the "allocated" can be much higher +than "used". Generally speaking, "peak" will more resemble the actual +memory consumed as reported by RSS, since returning memory to the OS is a hard +and not solved problem. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount of allocated + is much higher than the amount of used, we have large fragmentation issue + +* GC rawmalloced - large objects allocated with malloc. If this does not + correspond to the amount of RSS very well, consider using jemalloc as opposed + to system malloc + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + + .. _minimark-environment-variables: +Environment variables +--------------------- + PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: From pypy.commits at gmail.com Thu Feb 15 09:58:44 2018 From: pypy.commits at gmail.com (fijal) Date: Thu, 15 Feb 2018 06:58:44 -0800 (PST) Subject: [pypy-commit] pypy default: (arigo, fijal) improve the doc Message-ID: <5a85a024.1a7c1c0a.3b360.342a@mx.google.com> Author: fijal Branch: Changeset: r93825:bb02514372a2 Date: 2018-02-15 15:58 +0100 http://bitbucket.org/pypy/pypy/changeset/bb02514372a2/ Log: (arigo, fijal) improve the doc diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -15,14 +15,41 @@ processes) and cache sizes you might want to experiment with it via *PYPY_GC_NURSERY* environment variable. When the nursery is full, there is performed a minor collection. Freed objects are no longer referencable and -just die, without any effort, while surviving objects from the nursery -are copied to the old generation. Either to arenas, which are collections -of objects of the same size, or directly allocated with malloc if they're big -enough. +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) Since Incminimark is an incremental GC, the major collection is incremental, meaning there should not be any pauses longer than 1ms. + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + There is a special function in the ``gc`` module called ``get_stats(memory_pressure=False)``. @@ -56,19 +83,32 @@ In this particular case, which is just at startup, GC consumes relatively little memory and there is even less unused, but allocated memory. In case -there is a high memory fragmentation, the "allocated" can be much higher -than "used". Generally speaking, "peak" will more resemble the actual -memory consumed as reported by RSS, since returning memory to the OS is a hard -and not solved problem. +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. The details of various fields: -* GC in arenas - small old objects held in arenas. If the amount of allocated - is much higher than the amount of used, we have large fragmentation issue +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. -* GC rawmalloced - large objects allocated with malloc. If this does not - correspond to the amount of RSS very well, consider using jemalloc as opposed - to system malloc +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ * nursery - amount of memory allocated for nursery, fixed at startup, controlled via an environment variable @@ -91,7 +131,7 @@ ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` From pypy.commits at gmail.com Thu Feb 15 13:38:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 15 Feb 2018 10:38:20 -0800 (PST) Subject: [pypy-commit] pypy py3.5: gah, nonsense. I want an decode, not encode. add assert to catch this at test-time in the future Message-ID: <5a85d39c.d5171c0a.ce63.5496@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r93826:1e8c1d693fcf Date: 2018-02-15 19:28 +0100 http://bitbucket.org/pypy/pypy/changeset/1e8c1d693fcf/ Log: gah, nonsense. I want an decode, not encode. add assert to catch this at test-time in the future diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -148,6 +148,7 @@ # which never raises UnicodeEncodeError. Surrogate pairs are then # allowed, either paired or lone. A paired surrogate is considered # like the non-BMP character it stands for. See also *_utf8sp(). + assert isinstance(uni, unicode) return runicode.unicode_encode_utf_8( uni, len(uni), "strict", errorhandler=encode_error_handler(space), diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -4,7 +4,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.unicodehelper import encode_utf8 +from pypy.interpreter.unicodehelper import decode_utf8 from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string, @@ -432,8 +432,8 @@ def materialize_str_dict(self, space, obj, str_dict): new_obj = self.back.materialize_str_dict(space, obj, str_dict) if self.index == DICT: - enc_name = encode_utf8(space, self.name) - str_dict[enc_name] = obj._mapdict_read_storage(self.storageindex) + uni_name = decode_utf8(space, self.name) + str_dict[uni_name] = obj._mapdict_read_storage(self.storageindex) else: self._copy_attr(obj, new_obj) return new_obj From pypy.commits at gmail.com Fri Feb 16 03:27:53 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 16 Feb 2018 00:27:53 -0800 (PST) Subject: [pypy-commit] cffi default: In the setup.py, always import setuptools first on Windows Message-ID: <5a869609.54afdf0a.cb7dd.4028@mx.google.com> Author: Armin Rigo Branch: Changeset: r3097:b576c88f25f7 Date: 2018-02-16 09:27 +0100 http://bitbucket.org/cffi/cffi/changeset/b576c88f25f7/ Log: In the setup.py, always import setuptools first on Windows diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ import subprocess import errno +# on Windows we give up and always import setuptools early to fix things for us +if sys.platform == "win32": + import setuptools + sources = ['c/_cffi_backend.c'] libraries = ['ffi'] From pypy.commits at gmail.com Fri Feb 16 03:27:56 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 16 Feb 2018 00:27:56 -0800 (PST) Subject: [pypy-commit] cffi default: Implement ffi.dlclose() for the in-line case Message-ID: <5a86960c.410b1c0a.aea39.81e4@mx.google.com> Author: Armin Rigo Branch: Changeset: r3098:cdaebfeea0f0 Date: 2018-02-16 09:27 +0100 http://bitbucket.org/cffi/cffi/changeset/cdaebfeea0f0/ Log: Implement ffi.dlclose() for the in-line case diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3982,7 +3982,8 @@ static void dl_dealloc(DynLibObject *dlobj) { - dlclose(dlobj->dl_handle); + if (dlobj->dl_handle != NULL) + dlclose(dlobj->dl_handle); free(dlobj->dl_name); PyObject_Del(dlobj); } @@ -3992,6 +3993,17 @@ return PyText_FromFormat("", dlobj->dl_name); } +static int dl_check_closed(DynLibObject *dlobj) +{ + if (dlobj->dl_handle == NULL) + { + PyErr_Format(PyExc_ValueError, "library '%s' has already been closed", + dlobj->dl_name); + return -1; + } + return 0; +} + static PyObject *dl_load_function(DynLibObject *dlobj, PyObject *args) { CTypeDescrObject *ct; @@ -4002,6 +4014,9 @@ &CTypeDescr_Type, &ct, &funcname)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + if (!(ct->ct_flags & (CT_FUNCTIONPTR | CT_POINTER | CT_ARRAY))) { PyErr_Format(PyExc_TypeError, "function or pointer or array cdata expected, got '%s'", @@ -4034,6 +4049,9 @@ &CTypeDescr_Type, &ct, &varname)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + dlerror(); /* clear error condition */ data = dlsym(dlobj->dl_handle, varname); if (data == NULL) { @@ -4059,6 +4077,9 @@ &CTypeDescr_Type, &ct, &varname, &value)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + dlerror(); /* clear error condition */ data = dlsym(dlobj->dl_handle, varname); if (data == NULL) { @@ -4074,10 +4095,21 @@ return Py_None; } +static PyObject *dl_close_lib(DynLibObject *dlobj, PyObject *no_args) +{ + if (dl_check_closed(dlobj) < 0) + return NULL; + dlclose(dlobj->dl_handle); + dlobj->dl_handle = NULL; + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef dl_methods[] = { {"load_function", (PyCFunction)dl_load_function, METH_VARARGS}, {"read_variable", (PyCFunction)dl_read_variable, METH_VARARGS}, {"write_variable", (PyCFunction)dl_write_variable, METH_VARARGS}, + {"close_lib", (PyCFunction)dl_close_lib, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py --- a/testing/cffi0/test_function.py +++ b/testing/cffi0/test_function.py @@ -499,3 +499,23 @@ """) m = ffi.dlopen(lib_m) assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc', 'myvar'] + + def test_dlclose(self): + if self.Backend is CTypesBackend: + py.test.skip("not with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef("int foobar(void); int foobaz;") + lib = ffi.dlopen(lib_m) + ffi.dlclose(lib) + e = py.test.raises(ValueError, ffi.dlclose, lib) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobar') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobaz') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") From pypy.commits at gmail.com Fri Feb 16 04:00:53 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 16 Feb 2018 01:00:53 -0800 (PST) Subject: [pypy-commit] cffi default: Revert these small changes Message-ID: <5a869dc5.0f821c0a.ad16.efbd@mx.google.com> Author: Armin Rigo Branch: Changeset: r3099:d7d2eae6f698 Date: 2018-02-16 10:00 +0100 http://bitbucket.org/cffi/cffi/changeset/d7d2eae6f698/ Log: Revert these small changes diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3990,7 +3990,7 @@ static PyObject *dl_repr(DynLibObject *dlobj) { - return PyText_FromFormat("", dlobj->dl_name); + return PyText_FromFormat("", dlobj->dl_name); } static int dl_check_closed(DynLibObject *dlobj) @@ -4028,7 +4028,7 @@ if (funcptr == NULL) { const char *error = dlerror(); PyErr_Format(PyExc_AttributeError, - "function/symbol '%s' not found in library %s: %s", + "function/symbol '%s' not found in library '%s': %s", funcname, dlobj->dl_name, error); return NULL; } @@ -4058,7 +4058,7 @@ const char *error = dlerror(); if (error != NULL) { PyErr_Format(PyExc_KeyError, - "variable '%s' not found in library %s: %s", + "variable '%s' not found in library '%s': %s", varname, dlobj->dl_name, error); return NULL; } @@ -4085,7 +4085,7 @@ if (data == NULL) { const char *error = dlerror(); PyErr_Format(PyExc_KeyError, - "variable '%s' not found in library %s: %s", + "variable '%s' not found in library '%s': %s", varname, dlobj->dl_name, error); return NULL; } From pypy.commits at gmail.com Fri Feb 16 05:23:22 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 16 Feb 2018 02:23:22 -0800 (PST) Subject: [pypy-commit] cffi default: More Windows support for ffi.dlopen(), this time the out-of-line version Message-ID: <5a86b11a.2196df0a.a6111.976e@mx.google.com> Author: Armin Rigo Branch: Changeset: r3100:5448f4eca09c Date: 2018-02-16 11:22 +0100 http://bitbucket.org/cffi/cffi/changeset/5448f4eca09c/ Log: More Windows support for ffi.dlopen(), this time the out-of-line version diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4145,49 +4145,56 @@ dl_methods, /* tp_methods */ }; -static PyObject *b_load_library(PyObject *self, PyObject *args) -{ - char *filename_or_null, *printable_filename; - PyObject *s = NULL; +static void *b_do_dlopen(PyObject *args, char **p_printable_filename, + PyObject **p_temp) +{ + /* Logic to call the correct version of dlopen(). Returns NULL in case of error. + Otherwise, '*p_printable_filename' will point to a printable char version of + the filename (maybe utf-8-encoded). '*p_temp' will be set either to NULL or + to a temporary object that must be freed after looking at printable_filename. + */ void *handle; - DynLibObject *dlobj = NULL; + char *filename_or_null; int flags = 0; - + *p_temp = NULL; + if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { PyObject *dummy; if (!PyArg_ParseTuple(args, "|Oi:load_library", &dummy, &flags)) return NULL; filename_or_null = NULL; - printable_filename = ""; + *p_printable_filename = ""; } else { - printable_filename = NULL; - s = PyObject_Repr(PyTuple_GET_ITEM(args, 0)); - if (s != NULL) { - printable_filename = PyText_AsUTF8(s); - } - if (printable_filename == NULL) { - PyErr_Clear(); - printable_filename = "?"; - } - + PyObject *s = PyTuple_GET_ITEM(args, 0); #ifdef MS_WIN32 + Py_UNICODE *filenameW; + if (PyArg_ParseTuple(args, "u|i:load_library", &filenameW, &flags)) { - Py_UNICODE *filenameW; - if (PyArg_ParseTuple(args, "u|i:load_library", &filenameW, &flags)) - { - handle = dlopenW(filenameW); - goto got_handle; - } - PyErr_Clear(); - } +#if PY_MAJOR_VERSION < 3 + s = PyUnicode_AsUTF8String(s); + if (s == NULL) + return NULL; + *p_temp = s; +#endif + *p_printable_filename = PyText_AsUTF8(s); + if (*p_printable_filename == NULL) + return NULL; + + handle = dlopenW(filenameW); + goto got_handle; + } + PyErr_Clear(); #endif if (!PyArg_ParseTuple(args, "et|i:load_library", - Py_FileSystemDefaultEncoding, &filename_or_null, - &flags)) - goto error; + Py_FileSystemDefaultEncoding, &filename_or_null, &flags)) + return NULL; + + *p_printable_filename = PyText_AsUTF8(s); + if (*p_printable_filename == NULL) + return NULL; } if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) flags |= RTLD_NOW; @@ -4199,10 +4206,23 @@ #endif if (handle == NULL) { const char *error = dlerror(); - PyErr_Format(PyExc_OSError, "cannot load library %s: %s", - printable_filename, error); + PyErr_Format(PyExc_OSError, "cannot load library '%s': %s", + *p_printable_filename, error); + return NULL; + } + return handle; +} + +static PyObject *b_load_library(PyObject *self, PyObject *args) +{ + char *printable_filename; + PyObject *temp; + void *handle; + DynLibObject *dlobj = NULL; + + handle = b_do_dlopen(args, &printable_filename, &temp); + if (handle == NULL) goto error; - } dlobj = PyObject_New(DynLibObject, &dl_type); if (dlobj == NULL) { @@ -4211,9 +4231,9 @@ } dlobj->dl_handle = handle; dlobj->dl_name = strdup(printable_filename); - + error: - Py_XDECREF(s); + Py_XDECREF(temp); return (PyObject *)dlobj; } diff --git a/c/cdlopen.c b/c/cdlopen.c --- a/c/cdlopen.c +++ b/c/cdlopen.c @@ -40,35 +40,18 @@ static PyObject *ffi_dlopen(PyObject *self, PyObject *args) { - char *filename_or_null, *printable_filename; + char *modname; + PyObject *temp, *result = NULL; void *handle; - int flags = 0; - if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { - PyObject *dummy; - if (!PyArg_ParseTuple(args, "|Oi:load_library", - &dummy, &flags)) - return NULL; - filename_or_null = NULL; + handle = b_do_dlopen(args, &modname, &temp); + if (handle != NULL) + { + result = (PyObject *)lib_internal_new((FFIObject *)self, + modname, handle); } - else if (!PyArg_ParseTuple(args, "et|i:load_library", - Py_FileSystemDefaultEncoding, &filename_or_null, - &flags)) - return NULL; - - if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) - flags |= RTLD_NOW; - printable_filename = filename_or_null ? filename_or_null : ""; - - handle = dlopen(filename_or_null, flags); - if (handle == NULL) { - const char *error = dlerror(); - PyErr_Format(PyExc_OSError, "cannot load library '%s': %s", - printable_filename, error); - return NULL; - } - return (PyObject *)lib_internal_new((FFIObject *)self, - printable_filename, handle); + Py_XDECREF(temp); + return result; } static PyObject *ffi_dlclose(PyObject *self, PyObject *args) diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py --- a/testing/cffi1/test_re_python.py +++ b/testing/cffi1/test_re_python.py @@ -1,8 +1,9 @@ -import sys +import sys, os import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing from testing.udir import udir +from testing.support import u def setup_module(mod): @@ -35,6 +36,13 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + if sys.platform == "win32": + # test with a non-ascii char + outputfn1 = outputfilename + ofn, oext = os.path.splitext(outputfn1) + outputfilename = ofn + (u+'\u03be') + oext + #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) + os.rename(outputfn1, outputfilename) mod.extmod = outputfilename mod.tmpdir = tmpdir # @@ -107,12 +115,16 @@ from re_python_pysrc import ffi lib = ffi.dlopen(extmod) ffi.dlclose(lib) + if type(extmod) is not str: # unicode, on python 2 + str_extmod = extmod.encode('utf-8') + else: + str_extmod = extmod e = py.test.raises(ffi.error, ffi.dlclose, lib) assert str(e.value).startswith( - "library '%s' is already closed" % (extmod,)) + "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( - "library '%s' has been closed" % (extmod,)) + "library '%s' has been closed" % (str_extmod,)) def test_constant_via_lib(): from re_python_pysrc import ffi From pypy.commits at gmail.com Sat Feb 17 15:20:26 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 17 Feb 2018 12:20:26 -0800 (PST) Subject: [pypy-commit] pypy default: test, fix for tzinfo. datetime inherits from date, but date has no tzinfo, monkeypatch alloc appropriately Message-ID: <5a888e8a.42addf0a.f7dfd.aa25@mx.google.com> Author: Matti Picus Branch: Changeset: r93827:c7ea666563e3 Date: 2018-02-17 22:19 +0200 http://bitbucket.org/pypy/pypy/changeset/c7ea666563e3/ Log: test, fix for tzinfo. datetime inherits from date, but date has no tzinfo, monkeypatch alloc appropriately diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -128,6 +129,8 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + alloc=date_or_datetime_allocate, + attach=type_attach, dealloc=date_dealloc, ) @@ -136,10 +139,33 @@ attach=timedeltatype_attach, ) +def date_or_datetime_allocate(self, space, w_type, itemcount=0, immortal=False): + # allocates a date or datetime object. datetime has a tzinfo field, date does not + pytype = as_pyobj(space, w_type) + pytype = rffi.cast(PyTypeObjectPtr, pytype) + if w_type.name == 'date': + # XXX we should do this where the 'date' and 'datetime' type is instantiated + pytype.c_tp_basicsize = rffi.sizeof(PyObject.TO) + size = pytype.c_tp_basicsize + incref(space, pytype) + assert size >= rffi.sizeof(PyObject.TO) + buf = lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True, + add_memory_pressure=True, immortal=immortal) + pyobj = rffi.cast(PyObject, buf) + pyobj.c_ob_refcnt = 1 + #pyobj.c_ob_pypy_link should get assigned very quickly + pyobj.c_ob_type = pytype + return pyobj + def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return + # just make sure, should be removed + assert py_obj.c_ob_type.c_tp_basicsize > rffi.sizeof(PyObject.TO) py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -82,16 +82,6 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) - def test_tzinfo(self, space): - w_tzinfo = space.appexec( - [], """(): - from datetime import tzinfo - return tzinfo() - """) - assert PyTZInfo_Check(space, w_tzinfo) - assert PyTZInfo_CheckExact(space, w_tzinfo) - assert not PyTZInfo_Check(space, space.w_None) - class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -272,3 +262,81 @@ 2000, 6, 6, 6, 6, 6, 6) assert module.test_time_macros() == datetime.time(6, 6, 6, 6) assert module.test_delta_macros() == datetime.timedelta(6, 6, 6) + + def test_tzinfo(self): + module = self.import_extension('foo', [ + ("time_with_tzinfo", "METH_O", + """ PyDateTime_IMPORT; + return PyDateTimeAPI->Time_FromTime( + 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); + """), + ("datetime_with_tzinfo", "METH_O", + """ + PyObject * obj; + int tzrefcnt = args->ob_refcnt; + PyDateTime_IMPORT; + obj = PyDateTimeAPI->DateTime_FromDateAndTime( + 2000, 6, 6, 6, 6, 6, 6, args, + PyDateTimeAPI->DateTimeType); + if (!((PyDateTime_DateTime*)obj)->hastzinfo) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "missing tzinfo"); + return NULL; + } + if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not incremented"); + return NULL; + } + return obj; + + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time + # copied from datetime documentation + class GMT1(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + self.dst(dt) + def dst(self, dt): + return timedelta(0) + def tzname(self,dt): + return "GMT +1" + gmt1 = GMT1() + dt1 = module.time_with_tzinfo(gmt1) + assert dt1 == time(6, 6, 6, 6, gmt1) + assert '+01' in str(dt1) + assert module.datetime_with_tzinfo(gmt1) == datetime( + 2000, 6, 6, 6, 6, 6, 6, gmt1) + + def test_checks(self): + module = self.import_extension('foo', [ + ("checks", "METH_O", + """ PyDateTime_IMPORT; + return PyTuple_Pack(10, + PyBool_FromLong(PyDateTime_Check(args)), + PyBool_FromLong(PyDateTime_CheckExact(args)), + PyBool_FromLong(PyDate_Check(args)), + PyBool_FromLong(PyDate_CheckExact(args)), + PyBool_FromLong(PyTime_Check(args)), + PyBool_FromLong(PyTime_CheckExact(args)), + PyBool_FromLong(PyDelta_Check(args)), + PyBool_FromLong(PyDelta_CheckExact(args)), + PyBool_FromLong(PyTZInfo_Check(args)), + PyBool_FromLong(PyTZInfo_CheckExact(args)) + ); + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time, date + o = date(1, 1, 1) + assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6 + o = time(1, 1, 1) + assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4 + o = timedelta(1, 1, 1) + assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2 + o = datetime(1, 1, 1) + assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) + o = tzinfo() + assert module.checks(o) == (False,) * 8 + (True,) * 2 + From pypy.commits at gmail.com Sat Feb 17 15:54:35 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 17 Feb 2018 12:54:35 -0800 (PST) Subject: [pypy-commit] pypy default: find the correct place to modify the tp_basicsize for "date" Message-ID: <5a88968b.f4aedf0a.81d1e.ef9b@mx.google.com> Author: Matti Picus Branch: Changeset: r93828:15b387800a8e Date: 2018-02-17 22:53 +0200 http://bitbucket.org/pypy/pypy/changeset/15b387800a8e/ Log: find the correct place to modify the tp_basicsize for "date" diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -32,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -129,7 +133,6 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, - alloc=date_or_datetime_allocate, attach=type_attach, dealloc=date_dealloc, ) @@ -139,33 +142,12 @@ attach=timedeltatype_attach, ) -def date_or_datetime_allocate(self, space, w_type, itemcount=0, immortal=False): - # allocates a date or datetime object. datetime has a tzinfo field, date does not - pytype = as_pyobj(space, w_type) - pytype = rffi.cast(PyTypeObjectPtr, pytype) - if w_type.name == 'date': - # XXX we should do this where the 'date' and 'datetime' type is instantiated - pytype.c_tp_basicsize = rffi.sizeof(PyObject.TO) - size = pytype.c_tp_basicsize - incref(space, pytype) - assert size >= rffi.sizeof(PyObject.TO) - buf = lltype.malloc(rffi.VOIDP.TO, size, - flavor='raw', zero=True, - add_memory_pressure=True, immortal=immortal) - pyobj = rffi.cast(PyObject, buf) - pyobj.c_ob_refcnt = 1 - #pyobj.c_ob_pypy_link should get assigned very quickly - pyobj.c_ob_type = pytype - return pyobj - def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj ''' if space.type(w_obj).name == 'date': # No tzinfo return - # just make sure, should be removed - assert py_obj.c_ob_type.c_tp_basicsize > rffi.sizeof(PyObject.TO) py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): From pypy.commits at gmail.com Sat Feb 17 17:23:57 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 17 Feb 2018 14:23:57 -0800 (PST) Subject: [pypy-commit] pypy py3.5: be like cpython, maybe fix translation, tests on windows 8 and above Message-ID: <5a88ab7d.d5411c0a.c7166.fea8@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93829:7dc75691f071 Date: 2018-02-18 00:23 -0500 http://bitbucket.org/pypy/pypy/changeset/7dc75691f071/ Log: be like cpython, maybe fix translation, tests on windows 8 and above diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py --- a/rpython/rlib/rposix_environ.py +++ b/rpython/rlib/rposix_environ.py @@ -153,6 +153,8 @@ def envitems_llimpl(): environ = get_environ() result = [] + if not environ: + return result i = 0 while environ[i]: name_value = traits.charp2str(environ[i]) From pypy.commits at gmail.com Mon Feb 19 14:10:21 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Feb 2018 11:10:21 -0800 (PST) Subject: [pypy-commit] pypy py3.5: test, fix for compiling main.c, wmain.c if using precompiled headers Message-ID: <5a8b211d.c7c6df0a.1f037.8c79@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93830:f3013f4a60ea Date: 2018-02-19 19:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f3013f4a60ea/ Log: test, fix for compiling main.c, wmain.c if using precompiled headers diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py --- a/rpython/translator/platform/test/test_makefile.py +++ b/rpython/translator/platform/test/test_makefile.py @@ -85,7 +85,8 @@ txt = '#include \n' for i in range(ncfiles): txt += "int func%03d();\n" % i - txt += "\nint main(int argc, char * argv[])\n" + txt += "\n__declspec(dllexport) int\n" + txt += "pypy_main_startup(int argc, char * argv[])\n" txt += "{\n int i=0;\n" for i in range(ncfiles): txt += " i += func%03d();\n" % i @@ -119,7 +120,7 @@ clean = ('clean', '', 'rm -f $(OBJECTS) $(TARGET) ') get_time = time.time #write a non-precompiled header makefile - mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir) + mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir, shared=True) mk.rule(*clean) mk.write() t0 = get_time() @@ -128,7 +129,7 @@ t_normal = t1 - t0 self.platform.execute_makefile(mk, extra_opts=['clean']) # Write a super-duper makefile with precompiled headers - mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir, + mk = self.platform.gen_makefile(cfiles, eci, path=tmpdir, shared=True, headers_to_precompile=cfiles_precompiled_headers,) mk.rule(*clean) mk.write() diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -384,7 +384,8 @@ if len(headers_to_precompile)>0: if shared: - no_precompile_cfiles += ['main.c', 'wmain.c'] + no_precompile_cfiles += [m.makefile_dir / 'main.c', + m.makefile_dir / 'wmain.c'] stdafx_h = path.join('stdafx.h') txt = '#ifndef PYPY_STDAFX_H\n' txt += '#define PYPY_STDAFX_H\n' From pypy.commits at gmail.com Mon Feb 19 14:10:23 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Feb 2018 11:10:23 -0800 (PST) Subject: [pypy-commit] pypy py3.5: add externals/bin to PATH so tests pick up correct dlls Message-ID: <5a8b211f.984a1c0a.13fb1.e8d9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93831:b0c75af579f5 Date: 2018-02-19 21:09 +0200 http://bitbucket.org/pypy/pypy/changeset/b0c75af579f5/ Log: add externals/bin to PATH so tests pick up correct dlls diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -536,7 +536,13 @@ # The buildbot runs get_externals before building def patch_env(env, externals = Platform.externals): #print 'adding %s to PATH, INCLUDE, LIB' % basepath - env['PATH'] = externals + r'\bin;' + env.get('PATH', '') + binpath = externals + r'\bin' + path = os.environ['PATH'] + if binpath not in path.split(';'): + path = binpath + ';' + path + # make sure externals is in current path for tests and translating + os.environ['PATH'] = path + env['PATH'] = binpath + ';' + env.get('PATH', '') env['INCLUDE'] = externals + r'\include;' + env.get('INCLUDE', '') env['LIB'] = externals + r'\lib;' + env.get('LIB', '') return None From pypy.commits at gmail.com Mon Feb 19 14:34:36 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 11:34:36 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix test Message-ID: <5a8b26cc.4193df0a.d6e45.d0ce@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93832:5e9a14e1be53 Date: 2018-02-19 19:33 +0000 http://bitbucket.org/pypy/pypy/changeset/5e9a14e1be53/ Log: fix test diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -277,23 +277,23 @@ free(bytes); return NULL; } - result = PyString_FromStringAndSize((const char *)bytes, n); + result = PyBytes_FromStringAndSize((const char *)bytes, n); free(bytes); return result; """), ]) s = module.as_bytearray(0x41BC9A, 4, True, False) - assert s == "\x9A\xBC\x41\x00" + assert s == b"\x9A\xBC\x41\x00" s = module.as_bytearray(0x41BC9A, 4, False, False) - assert s == "\x00\x41\xBC\x9A" + assert s == b"\x00\x41\xBC\x9A" s = module.as_bytearray(0x41BC9A, 3, True, False) - assert s == "\x9A\xBC\x41" + assert s == b"\x9A\xBC\x41" s = module.as_bytearray(0x41BC9A, 3, True, True) - assert s == "\x9A\xBC\x41" + assert s == b"\x9A\xBC\x41" s = module.as_bytearray(0x9876, 2, True, False) - assert s == "\x76\x98" + assert s == b"\x76\x98" s = module.as_bytearray(0x9876 - 0x10000, 2, True, True) - assert s == "\x76\x98" + assert s == b"\x76\x98" raises(OverflowError, module.as_bytearray, 0x9876, 2, False, True) raises(OverflowError, module.as_bytearray, From pypy.commits at gmail.com Mon Feb 19 15:23:15 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 12:23:15 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix test (os.environ != posix.environ) Message-ID: <5a8b3233.89a2df0a.ef868.a271@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93833:f7cdc20cd320 Date: 2018-02-19 20:22 +0000 http://bitbucket.org/pypy/pypy/changeset/f7cdc20cd320/ Log: Fix test (os.environ != posix.environ) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1478,10 +1478,9 @@ def test_environ(self): import sys, os environ = os.environ - item_type = str if sys.platform.startswith('win') else bytes for k, v in environ.items(): - assert type(k) is item_type - assert type(v) is item_type + assert type(k) is str + assert type(v) is str name = next(iter(environ)) assert environ[name] is not None del environ[name] From pypy.commits at gmail.com Mon Feb 19 16:23:49 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Feb 2018 13:23:49 -0800 (PST) Subject: [pypy-commit] pypy winapi: close branch to be merged Message-ID: <5a8b4065.1a7c1c0a.3b360.c765@mx.google.com> Author: Matti Picus Branch: winapi Changeset: r93834:bc3564c15ff9 Date: 2018-02-19 23:16 -0500 http://bitbucket.org/pypy/pypy/changeset/bc3564c15ff9/ Log: close branch to be merged From pypy.commits at gmail.com Mon Feb 19 16:23:51 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Feb 2018 13:23:51 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge winapi which modernizes _winapi and _pypy_winbase_cffi Message-ID: <5a8b4067.42addf0a.f7dfd.1f80@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93835:4cf12d2f65d9 Date: 2018-02-19 23:21 -0500 http://bitbucket.org/pypy/pypy/changeset/4cf12d2f65d9/ Log: merge winapi which modernizes _winapi and _pypy_winbase_cffi diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -63,6 +63,12 @@ HANDLE hStdError; } STARTUPINFO, *LPSTARTUPINFO; +typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; +} SECURITY_ATTRIBUTES, *PSECURITY_ATTRIBUTES, *LPSECURITY_ATTRIBUTES; + typedef struct { HANDLE hProcess; HANDLE hThread; @@ -70,9 +76,41 @@ DWORD dwThreadId; } PROCESS_INFORMATION, *LPPROCESS_INFORMATION; +typedef struct _OVERLAPPED { + ULONG_PTR Internal; + ULONG_PTR InternalHigh; + union { + struct { + DWORD Offset; + DWORD OffsetHigh; + } DUMMYSTRUCTNAME; + PVOID Pointer; + } DUMMYUNIONNAME; + + HANDLE hEvent; +} OVERLAPPED, *LPOVERLAPPED; + + DWORD WINAPI GetVersion(void); BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD); +HANDLE WINAPI CreateNamedPipeA(LPCSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateNamedPipeW(LPWSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateFileA(LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD); +BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED); +HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR); +HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR); +VOID WINAPI SetEvent(HANDLE); +BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED); BOOL WINAPI CloseHandle(HANDLE); +DWORD WINAPI GetLastError(VOID); +BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL); + HANDLE WINAPI GetCurrentProcess(void); BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0), - _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), - _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xAB\x03\x00\x00\x13\x11\x00\x00\xB0\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\xAA\x03\x00\x00\xA8\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x29\x11\x00\x00\x18\x03\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x6B\x03\x00\x00\x49\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x33\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x66\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\xA9\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xAB\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x6E\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6B\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x77\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xB0\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\x05\x09\x00\x00\x03\x09\x00\x00\x02\x01\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xAF\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x27\x23CancelIoEx',0,b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x27\x23ConnectNamedPipe',0,b'\x00\x00\x6D\x23CreateEventA',0,b'\x00\x00\x73\x23CreateEventW',0,b'\x00\x00\x79\x23CreateFileA',0,b'\x00\x00\x9B\x23CreateFileW',0,b'\x00\x00\x82\x23CreateNamedPipeA',0,b'\x00\x00\x91\x23CreateNamedPipeW',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x48\x23CreateProcessW',0,b'\x00\x00\x3F\x23DuplicateHandle',0,b'\x00\x00\x8F\x23GetCurrentProcess',0,b'\x00\x00\x35\x23GetExitCodeProcess',0,b'\x00\x00\x63\x23GetLastError',0,b'\x00\x00\x5E\x23GetModuleFileNameW',0,b'\x00\x00\x2B\x23GetOverlappedResult',0,b'\x00\x00\x8C\x23GetStdHandle',0,b'\x00\x00\x63\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x57\x23SetErrorMode',0,b'\x00\x00\xA4\x23SetEvent',0,b'\x00\x00\x39\x23SetNamedPipeHandleState',0,b'\x00\x00\x31\x23TerminateProcess',0,b'\x00\x00\x5A\x23WaitForSingleObject',0,b'\x00\x00\x54\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x68\x23_getwch',0,b'\x00\x00\x68\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x6A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x65\x23_ungetwch',0), + _struct_unions = ((b'\x00\x00\x00\xAD\x00\x00\x00\x03$1',b'\x00\x00\xAC\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x00\xAC\x00\x00\x00\x02$2',b'\x00\x00\x18\x11Offset',b'\x00\x00\x18\x11OffsetHigh'),(b'\x00\x00\x00\xA8\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\xAA\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x66\x11wShowWindow',b'\x00\x00\x66\x11cbReserved2',b'\x00\x00\xAE\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x00\xA7\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x18\x11Internal',b'\x00\x00\x18\x11InternalHigh',b'\x00\x00\xAD\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x00\xA9\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x18\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle')), + _typenames = (b'\x00\x00\x00\x29LPOVERLAPPED',b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x6ELPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\xA7OVERLAPPED',b'\x00\x00\x00\xA8PROCESS_INFORMATION',b'\x00\x00\x00\x6EPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xA9SECURITY_ATTRIBUTES',b'\x00\x00\x00\xAASTARTUPINFO',b'\x00\x00\x00\x66wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -1,12 +1,12 @@ """ -Support routines for subprocess module. +Support routines for subprocess and multiprocess module. Currently, this extension module is only required when using the -subprocess module on Windows. +modules on Windows. """ import sys if sys.platform != 'win32': - raise ImportError("The '_subprocess' module is only available on Windows") + raise ImportError("The '_winapi' module is only available on Windows") # Declare external Win32 functions @@ -14,7 +14,7 @@ _kernel32 = _ffi.dlopen('kernel32') GetVersion = _kernel32.GetVersion - +NULL = _ffi.NULL # Now the _subprocess module implementation @@ -33,13 +33,116 @@ def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") - res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size) + res = _kernel32.CreatePipe(handles, handles + 1, NULL, size) if not res: raise _WinError() return _handle2int(handles[0]), _handle2int(handles[1]) +def CreateNamedPipe(*args): + handle = _kernel32.CreateNamedPipeW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def CreateFile(*args): + handle = _kernel32.CreateFileW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def SetNamedPipeHandleState(namedpipe, mode, max_collection_count, collect_data_timeout): + d0 = _ffi.new('DWORD[1]', [mode]) + if max_collection_count is None: + d1 = NULL + else: + d1 = _ffi.new('DWORD[1]', [max_collection_count]) + if collect_data_timeout is None: + d2 = NULL + else: + d2 = _ffi.new('DWORD[1]', [collect_data_timeout]) + ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2) + if not ret: + raise _WinError() + +class Overlapped(object): + def __init__(self, handle): + self.overlapped = _ffi.new('OVERLAPPED[1]') + self.handle = handle + self.readbuffer = None + self.pending = 0 + self.completed = 0 + self.writebuffer = None + self.overlapped[0].hEvent = \ + _kernel32.CreateEventW(NULL, True, False, NULL) + + def __del__(self): + # do this somehow else + xxx + err = _kernel32.GetLastError() + bytes = _ffi.new('DWORD[1]') + o = overlapped[0] + if overlapped[0].pending: + if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ + self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): + # The operation is no longer pending, nothing to do + pass + else: + raise RuntimeError('deleting an overlapped strucwith a pending operation not supported') + + @property + def event(self): + return None + + def GetOverlappedResult(self, wait): + transferred = _ffi.new('DWORD[1]', [0]) + res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) + if not res: + res = GetLastError() + if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED): + self.completed = 1 + self.pending = 0 + elif res == ERROR_IO_INCOMPLETE: + pass + else: + self.pending = 0 + raise _WinError() + if self.completed and self.read_buffer: + if transferred != len(self.read_buffer): + raise _WinError() + return transferred[0], err + + def getbuffer(self): + xxx + return None + + def cancel(self): + xxx + return None + + +def ConnectNamedPipe(handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + if overlapped: + # Overlapped ConnectNamedPipe never returns a success code + assert success == 0 + err = _kernel32.GetLastError() + if err == ERROR_IO_PENDING: + overlapped[0].pending = 1 + elif err == ERROR_PIPE_CONNECTED: + _kernel32.SetEvent(ov.overlapped[0].hEvent) + else: + del ov + raise _WinError() + return ov + elif not success: + raise _WinError() + def GetCurrentProcess(): return _handle2int(_kernel32.GetCurrentProcess()) @@ -155,6 +258,7 @@ raise _WinError() return _ffi.string(buf) +# #define macros from WinBase.h and elsewhere STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -171,3 +275,52 @@ CREATE_UNICODE_ENVIRONMENT = 0x400 STILL_ACTIVE = 259 _MAX_PATH = 260 + +ERROR_SUCCESS = 0 +ERROR_NETNAME_DELETED = 64 +ERROR_BROKEN_PIPE = 109 +ERROR_MORE_DATA = 234 +ERROR_PIPE_CONNECTED = 535 +ERROR_OPERATION_ABORTED = 995 +ERROR_IO_INCOMPLETE = 996 +ERROR_IO_PENDING = 997 + +PIPE_ACCESS_INBOUND = 0x00000001 +PIPE_ACCESS_OUTBOUND = 0x00000002 +PIPE_ACCESS_DUPLEX = 0x00000003 +PIPE_WAIT = 0x00000000 +PIPE_NOWAIT = 0x00000001 +PIPE_READMODE_BYTE = 0x00000000 +PIPE_READMODE_MESSAGE = 0x00000002 +PIPE_TYPE_BYTE = 0x00000000 +PIPE_TYPE_MESSAGE = 0x00000004 +PIPE_ACCEPT_REMOTE_CLIENTS = 0x00000000 +PIPE_REJECT_REMOTE_CLIENTS = 0x00000008 + +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +GENERIC_EXECUTE= 0x20000000 +GENERIC_ALL = 0x10000000 +INVALID_HANDLE_VALUE = -1 +FILE_FLAG_WRITE_THROUGH = 0x80000000 +FILE_FLAG_OVERLAPPED = 0x40000000 +FILE_FLAG_NO_BUFFERING = 0x20000000 +FILE_FLAG_RANDOM_ACCESS = 0x10000000 +FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 +FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 +FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 +FILE_FLAG_POSIX_SEMANTICS = 0x01000000 +FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +FILE_FLAG_OPEN_NO_RECALL = 0x00100000 +FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000 + +NMPWAIT_WAIT_FOREVER = 0xffffffff +NMPWAIT_NOWAIT = 0x00000001 +NMPWAIT_USE_DEFAULT_WAIT = 0x00000000 + +CREATE_NEW = 1 +CREATE_ALWAYS = 2 +OPEN_EXISTING = 3 +OPEN_ALWAYS = 4 +TRUNCATE_EXISTING = 5 + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -44,3 +44,7 @@ .. branch: msvc14 Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: winapi + +Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 From pypy.commits at gmail.com Mon Feb 19 16:44:13 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Feb 2018 13:44:13 -0800 (PST) Subject: [pypy-commit] pypy py3.5: revert 5a84b83fffbe, breaks finding vs2017 Message-ID: <5a8b452d.4bb21c0a.bd3af.52e1@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93836:82a463d6b79a Date: 2018-02-19 23:41 -0500 http://bitbucket.org/pypy/pypy/changeset/82a463d6b79a/ Log: revert 5a84b83fffbe, breaks finding vs2017 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -55,10 +55,7 @@ vcdict = None toolsdir = None try: - if vsver < 140: - toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] - else: - raise KeyError('always use registry values') + toolsdir = os.environ['VS%sCOMNTOOLS' % vsver] except KeyError: # use setuptools from python3 to find tools try: From pypy.commits at gmail.com Mon Feb 19 21:39:57 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 18:39:57 -0800 (PST) Subject: [pypy-commit] pypy default: Move pyrepl tests to extra_tests/ Message-ID: <5a8b8a7d.50131c0a.8fb42.eeee@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93837:87d144ad39a4 Date: 2018-02-20 02:39 +0000 http://bitbucket.org/pypy/pypy/changeset/87d144ad39a4/ Log: Move pyrepl tests to extra_tests/ diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py rename from pypy/module/test_lib_pypy/pyrepl/test_functional.py rename to extra_tests/test_pyrepl/test_functional.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py From pypy.commits at gmail.com Mon Feb 19 21:46:43 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 18:46:43 -0800 (PST) Subject: [pypy-commit] pypy default: Update pytest fixture syntax Message-ID: <5a8b8c13.0fe71c0a.2490d.9591@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93838:518bdb4a094d Date: 2018-02-20 02:46 +0000 http://bitbucket.org/pypy/pypy/changeset/518bdb4a094d/ Log: Update pytest fixture syntax diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py --- a/extra_tests/test_pyrepl/test_functional.py +++ b/extra_tests/test_pyrepl/test_functional.py @@ -7,7 +7,8 @@ import sys -def pytest_funcarg__child(request): + at pytest.fixture() +def child(): try: import pexpect except ImportError: From pypy.commits at gmail.com Mon Feb 19 22:10:53 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 19:10:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5a8b91bd.cb951c0a.821c9.e455@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93839:dd3b9bfab6a0 Date: 2018-02-20 02:47 +0000 http://bitbucket.org/pypy/pypy/changeset/dd3b9bfab6a0/ Log: hg merge default diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py rename from pypy/module/test_lib_pypy/pyrepl/test_functional.py rename to extra_tests/test_pyrepl/test_functional.py --- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py +++ b/extra_tests/test_pyrepl/test_functional.py @@ -7,7 +7,8 @@ import sys -def pytest_funcarg__child(request): + at pytest.fixture() +def child(): try: import pexpect except ImportError: diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -31,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -128,6 +133,7 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + attach=type_attach, dealloc=date_dealloc, ) @@ -138,8 +144,10 @@ def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -82,16 +82,6 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) - def test_tzinfo(self, space): - w_tzinfo = space.appexec( - [], """(): - from datetime import tzinfo - return tzinfo() - """) - assert PyTZInfo_Check(space, w_tzinfo) - assert PyTZInfo_CheckExact(space, w_tzinfo) - assert not PyTZInfo_Check(space, space.w_None) - class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -272,3 +262,81 @@ 2000, 6, 6, 6, 6, 6, 6) assert module.test_time_macros() == datetime.time(6, 6, 6, 6) assert module.test_delta_macros() == datetime.timedelta(6, 6, 6) + + def test_tzinfo(self): + module = self.import_extension('foo', [ + ("time_with_tzinfo", "METH_O", + """ PyDateTime_IMPORT; + return PyDateTimeAPI->Time_FromTime( + 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); + """), + ("datetime_with_tzinfo", "METH_O", + """ + PyObject * obj; + int tzrefcnt = args->ob_refcnt; + PyDateTime_IMPORT; + obj = PyDateTimeAPI->DateTime_FromDateAndTime( + 2000, 6, 6, 6, 6, 6, 6, args, + PyDateTimeAPI->DateTimeType); + if (!((PyDateTime_DateTime*)obj)->hastzinfo) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "missing tzinfo"); + return NULL; + } + if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not incremented"); + return NULL; + } + return obj; + + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time + # copied from datetime documentation + class GMT1(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + self.dst(dt) + def dst(self, dt): + return timedelta(0) + def tzname(self,dt): + return "GMT +1" + gmt1 = GMT1() + dt1 = module.time_with_tzinfo(gmt1) + assert dt1 == time(6, 6, 6, 6, gmt1) + assert '+01' in str(dt1) + assert module.datetime_with_tzinfo(gmt1) == datetime( + 2000, 6, 6, 6, 6, 6, 6, gmt1) + + def test_checks(self): + module = self.import_extension('foo', [ + ("checks", "METH_O", + """ PyDateTime_IMPORT; + return PyTuple_Pack(10, + PyBool_FromLong(PyDateTime_Check(args)), + PyBool_FromLong(PyDateTime_CheckExact(args)), + PyBool_FromLong(PyDate_Check(args)), + PyBool_FromLong(PyDate_CheckExact(args)), + PyBool_FromLong(PyTime_Check(args)), + PyBool_FromLong(PyTime_CheckExact(args)), + PyBool_FromLong(PyDelta_Check(args)), + PyBool_FromLong(PyDelta_CheckExact(args)), + PyBool_FromLong(PyTZInfo_Check(args)), + PyBool_FromLong(PyTZInfo_CheckExact(args)) + ); + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time, date + o = date(1, 1, 1) + assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6 + o = time(1, 1, 1) + assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4 + o = timedelta(1, 1, 1) + assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2 + o = datetime(1, 1, 1) + assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) + o = tzinfo() + assert module.checks(o) == (False,) * 8 + (True,) * 2 + diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -72,7 +72,7 @@ def __repr__(self): if self._s.total_memory_pressure != -1: - extra = "\nmemory pressure: %s" % self.total_memory_pressure + extra = "\n memory pressure: %s" % self.total_memory_pressure else: extra = "" return """Total memory consumed: @@ -109,5 +109,5 @@ self.memory_allocated_sum) -def get_stats(): - return GcStats(gc._get_stats()) +def get_stats(memory_pressure=False): + return GcStats(gc._get_stats(memory_pressure=memory_pressure)) From pypy.commits at gmail.com Mon Feb 19 22:10:55 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 19 Feb 2018 19:10:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix tests Message-ID: <5a8b91bf.1b88df0a.5b0f.f70a@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93840:a5f69bc67c34 Date: 2018-02-20 03:10 +0000 http://bitbucket.org/pypy/pypy/changeset/a5f69bc67c34/ Log: fix tests diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -102,8 +102,8 @@ assert len(done) == 5 def test_weakrefable(self): - import thread, weakref - weakref.ref(thread._local()) + import _thread, weakref + weakref.ref(_thread._local()) def test_local_is_not_immortal(self): import _thread, gc, time diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -53,8 +53,8 @@ assert lock.locked() is False def test_weakrefable(self): - import thread, weakref - weakref.ref(thread.allocate_lock()) + import _thread, weakref + weakref.ref(_thread.allocate_lock()) def test_timeout(self): import _thread From pypy.commits at gmail.com Tue Feb 20 01:37:15 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 19 Feb 2018 22:37:15 -0800 (PST) Subject: [pypy-commit] cffi default: silence a gcc warning shown in some (recent?) versions of gcc Message-ID: <5a8bc21b.93911c0a.fbf52.bbd8@mx.google.com> Author: Armin Rigo Branch: Changeset: r3101:8c670d218355 Date: 2018-02-20 07:37 +0100 http://bitbucket.org/cffi/cffi/changeset/8c670d218355/ Log: silence a gcc warning shown in some (recent?) versions of gcc diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3794,27 +3794,29 @@ static int check_bytes_for_float_compatible(PyObject *io, double *out_value) { if (PyBytes_Check(io)) { - if (PyBytes_GET_SIZE(io) != 1) { - Py_DECREF(io); - return -1; - } + if (PyBytes_GET_SIZE(io) != 1) + goto error; *out_value = (unsigned char)PyBytes_AS_STRING(io)[0]; return 1; } else if (PyUnicode_Check(io)) { char ignored[80]; cffi_char32_t ordinal; - if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) { - Py_DECREF(io); - return -1; - } + if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) + goto error; /* the signness of the 32-bit version of wide chars should not * matter here, because 'ordinal' comes from a normal Python * unicode string */ *out_value = ordinal; return 1; } + *out_value = 0; /* silence a gcc warning if this function is inlined */ return 0; + + error: + Py_DECREF(io); + *out_value = 0; /* silence a gcc warning if this function is inlined */ + return -1; } static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob) From pypy.commits at gmail.com Tue Feb 20 12:46:43 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Feb 2018 09:46:43 -0800 (PST) Subject: [pypy-commit] pypy throw-away: try to debug win32 buildbot Message-ID: <5a8c5f03.1a7c1c0a.3b360.2375@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93841:0ed68c3e6d03 Date: 2018-02-20 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/0ed68c3e6d03/ Log: try to debug win32 buildbot diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -83,6 +83,7 @@ stderr=subprocess.PIPE) stdout, stderr = popen.communicate() + print 'running', vcvars, "\nstderr '%s'" % stderr, '\nstdout' if popen.wait() != 0: return None except: @@ -92,6 +93,7 @@ vcdict = {} for line in stdout.split("\n"): if '=' not in line: + print line continue key, value = line.split('=', 1) vcdict[key] = value @@ -277,7 +279,10 @@ mfid = 2 out_arg = '-outputresource:%s;%s' % (exe_name, mfid) args = ['-nologo', '-manifest', str(temp_manifest), out_arg] - self._execute_c_compiler('mt.exe', args, exe_name) + try: + self._execute_c_compiler('mt.exe', args, exe_name) + except EnvironmentError: + print '\n'.join(os.environ['PATH'].split(';')) return exe_name From pypy.commits at gmail.com Tue Feb 20 13:08:00 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Feb 2018 10:08:00 -0800 (PST) Subject: [pypy-commit] pypy throw-away: print the path used by mt.exe Message-ID: <5a8c6400.172f1c0a.5a69a.5ef4@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93842:b11531b4a8a8 Date: 2018-02-20 20:06 -0500 http://bitbucket.org/pypy/pypy/changeset/b11531b4a8a8/ Log: print the path used by mt.exe diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -282,7 +282,8 @@ try: self._execute_c_compiler('mt.exe', args, exe_name) except EnvironmentError: - print '\n'.join(os.environ['PATH'].split(';')) + print '\n'.join(self.c_environ['PATH'].split(';')) + raise return exe_name From pypy.commits at gmail.com Tue Feb 20 13:40:02 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Feb 2018 10:40:02 -0800 (PST) Subject: [pypy-commit] pypy throw-away: print directory contents where mt.exe should be Message-ID: <5a8c6b82.86c11c0a.2f567.a5cb@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93843:3ae51739f88f Date: 2018-02-20 20:39 -0500 http://bitbucket.org/pypy/pypy/changeset/3ae51739f88f/ Log: print directory contents where mt.exe should be diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -282,7 +282,12 @@ try: self._execute_c_compiler('mt.exe', args, exe_name) except EnvironmentError: - print '\n'.join(self.c_environ['PATH'].split(';')) + paths = self.c_environ['PATH'].split(';')) + print 'paths, looking for mt.exe' + for p in paths: + print p + if 'Kit' in p: + print ' \n'.join(os.listdir(p)) raise return exe_name From pypy.commits at gmail.com Tue Feb 20 13:49:44 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Feb 2018 10:49:44 -0800 (PST) Subject: [pypy-commit] pypy throw-away: typo Message-ID: <5a8c6dc8.0ea7df0a.149c7.8295@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93844:f6583258a1eb Date: 2018-02-20 20:48 -0500 http://bitbucket.org/pypy/pypy/changeset/f6583258a1eb/ Log: typo diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -282,7 +282,7 @@ try: self._execute_c_compiler('mt.exe', args, exe_name) except EnvironmentError: - paths = self.c_environ['PATH'].split(';')) + paths = self.c_environ['PATH'].split(';') print 'paths, looking for mt.exe' for p in paths: print p From pypy.commits at gmail.com Tue Feb 20 15:22:48 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Feb 2018 12:22:48 -0800 (PST) Subject: [pypy-commit] pypy refactor-slots: Close branch refactor-slots Message-ID: <5a8c8398.8ebadf0a.a4986.0fa0@mx.google.com> Author: Ronan Lamy Branch: refactor-slots Changeset: r93845:acf679554bd0 Date: 2018-02-20 20:22 +0000 http://bitbucket.org/pypy/pypy/changeset/acf679554bd0/ Log: Close branch refactor-slots From pypy.commits at gmail.com Tue Feb 20 15:23:08 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Feb 2018 12:23:08 -0800 (PST) Subject: [pypy-commit] pypy default: Merged refactor-slots into default Message-ID: <5a8c83ac.4dc5df0a.cf770.b893@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93846:29f781e422ce Date: 2018-02-20 20:22 +0000 http://bitbucket.org/pypy/pypy/changeset/29f781e422ce/ Log: Merged refactor-slots into default diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -6,7 +6,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, - Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) + Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, @@ -143,7 +143,6 @@ def wrap_inquirypred(space, w_self, w_args, func): func_inquiry = rffi.cast(inquiry, func) check_num_args(space, w_args, 0) - args_w = space.fixedview(w_args) res = generic_cpy_call(space, func_inquiry, w_self) res = rffi.cast(lltype.Signed, res) if res == -1: @@ -423,285 +422,331 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): + +SLOTS = {} + at specialize.memo() +def get_slot_tp_function(space, typedef, name, method_name): + """Return a description of the slot C function to use for the built-in + type for 'typedef'. The 'name' is the slot name. This is a memo + function that, after translation, returns one of a built-in finite set. + """ + key = (typedef, name) + try: + return SLOTS[key] + except KeyError: + slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) + llfunc = llslot(space, slot_func) if slot_func else None + SLOTS[key] = llfunc + return llfunc + + +def make_unary_slot(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - - handled = False - # unary functions - for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_as_number.c_nb_hex', '__hex__'), - ('tp_as_number.c_nb_oct', '__oct__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('next') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set - else: - # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce - # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length - # richcmpfunc(s) + slot_fn = w_type.lookup(attr) + if slot_fn is None: return + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) return slot_func +UNARY_SLOTS = [ + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + + at slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + + at slot_factory('tp_as_mapping.c_mp_ass_subscript') +def make_sq_set_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, w_arg1, w_arg2) + else: + space.call_function(slot_del, w_self, w_arg1) + return 0 + return slot_func + + + at slot_factory('tp_as_sequence.c_sq_ass_item') +def make_sq_ass_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) + else: + space.call_function(slot_del, w_self, space.newint(arg1)) + return 0 + return slot_func + +def make_tp_setattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + setattr_fn = w_type.lookup('__setattr__') + delattr_fn = w_type.lookup('__delattr__') + if setattr_fn is None or delattr_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, + error=-1) + @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) + def slot_tp_setattro(space, w_self, w_name, w_value): + if w_value is not None: + space.call_function(setattr_fn, w_self, w_name, w_value) + else: + space.call_function(delattr_fn, w_self, w_name) + return 0 + return slot_tp_setattro + + at slot_factory('tp_getattro') +def make_tp_getattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + getattr_fn = w_type.lookup('__getattribute__') + if getattr_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) + def slot_tp_getattro(space, w_self, w_name): + return space.call_function(getattr_fn, w_self, w_name) + return slot_tp_getattro + + at slot_factory('tp_call') +def make_tp_call(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + call_fn = w_type.lookup('__call__') + if call_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(call_fn, args) + return slot_tp_call + + at slot_factory('tp_iternext') +def make_tp_iternext(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + iternext_fn = w_type.lookup('next') + if iternext_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_iternext(space, w_self): + try: + return space.call_function(iternext_fn, w_self) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + return None + return slot_tp_iternext + + at slot_factory('tp_init') +def make_tp_init(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + init_fn = w_type.lookup('__init__') + if init_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_init(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + space.call_args(init_fn, args) + return 0 + return slot_tp_init + + at slot_factory('tp_new') +def make_tp_new(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + new_fn = w_type.lookup('__new__') + if new_fn is None: + return + + @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_new(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(space.get(new_fn, w_self), args) + return slot_tp_new + + at slot_factory('tp_as_buffer.c_bf_getbuffer') +def make_bf_getbuffer(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + buff_fn = w_type.lookup('__buffer__') + if buff_fn is not None: + return slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + return slot_from_buffer_w(space, typedef) + else: + return None + + at slot_factory('tp_descr_get') +def make_tp_descr_get(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + get_fn = w_type.lookup('__get__') + if get_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_get(space, w_self, w_obj, w_value): + if w_obj is None: + w_obj = space.w_None + return space.call_function(get_fn, w_self, w_obj, w_value) + return slot_tp_descr_get + + at slot_factory('tp_descr_set') +def make_tp_descr_set(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + set_fn = w_type.lookup('__set__') + delete_fn = w_type.lookup('__delete__') + if set_fn is None and delete_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_set(space, w_self, w_obj, w_value): + if w_value is not None: + if set_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __set__", typedef.name) + space.call_function(set_fn, w_self, w_obj, w_value) + else: + if delete_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __delete__", typedef.name) + space.call_function(delete_fn, w_self, w_obj) + return 0 + return slot_tp_descr_set + def slot_from___buffer__(space, typedef, buff_fn): name = 'bf_getbuffer' @@ -730,7 +775,7 @@ return 0 return buff_w -def slot_from_buffer_w(space, typedef, buff_fn): +def slot_from_buffer_w(space, typedef): name = 'bf_getbuffer' @slot_function([PyObject, Py_bufferP, rffi.INT_real], rffi.INT_real, error=-1) @@ -765,6 +810,28 @@ missing_wrapper.__name__ = name globals()[name] = missing_wrapper +def make_missing_slot(space, typedef, name, attr): + return None + +missing_builtin_slots = [ + 'tp_print', 'tp_compare', 'tp_getattr', 'tp_setattr', 'tp_setattro', + 'tp_richcompare', 'tp_del', 'tp_as_buffer.c_bf_getwritebuffer', + 'tp_as_number.c_nb_nonzero', 'tp_as_number.c_nb_coerce', + 'tp_as_number.c_nb_inplace_add', 'tp_as_number.c_nb_inplace_subtract', + 'tp_as_number.c_nb_inplace_multiply', 'tp_as_number.c_nb_inplace_divide', + 'tp_as_number.c_nb_inplace_remainder', 'tp_as_number.c_nb_inplace_power', + 'tp_as_number.c_nb_inplace_lshift', 'tp_as_number.c_nb_inplace_rshift', + 'tp_as_number.c_nb_inplace_and', 'tp_as_number.c_nb_inplace_xor', + 'tp_as_number.c_nb_inplace_or', + 'tp_as_number.c_nb_floor_divide', 'tp_as_number.c_nb_true_divide', + 'tp_as_number.c_nb_inplace_floor_divide', 'tp_as_number.c_nb_inplace_true_divide', + 'tp_as_sequence.c_sq_slice', 'tp_as_sequence.c_sq_ass_slice', + 'tp_as_sequence.c_sq_contains', + 'tp_as_buffer.c_bf_getreadbuffer', + ] +for name in missing_builtin_slots: + slot_factory(name)(make_missing_slot) + PyWrapperFlag_KEYWORDS = 1 diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -31,7 +31,7 @@ PyObject, make_ref, from_ref, get_typedescr, make_typedescr, track_reference, decref, as_pyobj) from pypy.module.cpyext.slotdefs import ( - slotdefs_for_tp_slots, slotdefs_for_wrappers, build_slot_tp_function, + slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function, llslot) from pypy.module.cpyext.state import State from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne @@ -226,22 +226,6 @@ dict_w[name] = w_descr i += 1 -SLOTS = {} - at specialize.memo() -def get_slot_tp_function(space, typedef, name): - """Return a description of the slot C function to use for the built-in - type for 'typedef'. The 'name' is the slot name. This is a memo - function that, after translation, returns one of a built-in finite set. - """ - key = (typedef, name) - try: - return SLOTS[key] - except KeyError: - slot_func = build_slot_tp_function(space, typedef, name) - api_func = slot_func.api_func if slot_func else None - SLOTS[key] = api_func - return api_func - missing_slots={} def warn_missing_slot(space, method_name, slot_name, w_type): if not we_are_translated(): @@ -281,12 +265,11 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_apifunc = get_slot_tp_function(space, typedef, slot_name) - if not slot_apifunc: + slot_llfunc = get_slot_tp_function(space, typedef, slot_name, method_name) + if not slot_llfunc: warn_missing_slot(space, method_name, slot_name, w_type) continue - slot_func_helper = slot_apifunc.get_llhelper(space) - fill_slot(space, pto, w_type, slot_names, slot_func_helper) + fill_slot(space, pto, w_type, slot_names, slot_llfunc) @specialize.arg(3) def fill_slot(space, pto, w_type, slot_names, slot_func_helper): @@ -332,7 +315,7 @@ def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented - hash_not_impl = PyObject_HashNotImplemented.api_func.get_llhelper(space) + hash_not_impl = llslot(space, PyObject_HashNotImplemented) for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue From pypy.commits at gmail.com Tue Feb 20 15:23:38 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Feb 2018 12:23:38 -0800 (PST) Subject: [pypy-commit] pypy py3.5-refactor-slots: Close branch py3.5-refactor-slots Message-ID: <5a8c83ca.8281df0a.64966.dbd7@mx.google.com> Author: Ronan Lamy Branch: py3.5-refactor-slots Changeset: r93847:0aa74491da23 Date: 2018-02-20 20:23 +0000 http://bitbucket.org/pypy/pypy/changeset/0aa74491da23/ Log: Close branch py3.5-refactor-slots From pypy.commits at gmail.com Tue Feb 20 15:23:56 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Feb 2018 12:23:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Merged py3.5-refactor-slots into py3.5 Message-ID: <5a8c83dc.c83f1c0a.ea750.9b1b@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93848:53200cc06c53 Date: 2018-02-20 20:23 +0000 http://bitbucket.org/pypy/pypy/changeset/53200cc06c53/ Log: Merged py3.5-refactor-slots into py3.5 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -411,286 +411,327 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): - w_type = space.gettypeobject(typedef) - handled = False - # unary functions - for tp_name, attr in [('tp_as_async.c_am_await', '__await__'), - ('tp_as_async.c_am_anext', '__anext__'), - ('tp_as_async.c_am_aiter', '__aiter__'), - ('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('__next__') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set +def build_slot_tp_function(space, typedef, name, method_name): + if name in SLOT_FACTORIES: + return SLOT_FACTORIES[name](space, typedef, name, method_name) else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length # richcmpfunc(s) return + +def make_unary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) return slot_func +UNARY_SLOTS = [ + 'tp_as_async.c_am_await', + 'tp_as_async.c_am_anext', + 'tp_as_async.c_am_aiter', + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + + at slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + + at slot_factory('tp_as_mapping.c_mp_ass_subscript') +def make_sq_set_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, w_arg1, w_arg2) + else: + space.call_function(slot_del, w_self, w_arg1) + return 0 + return slot_func + + + at slot_factory('tp_as_sequence.c_sq_ass_item') +def make_sq_ass_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) + else: + space.call_function(slot_del, w_self, space.newint(arg1)) + return 0 + return slot_func + +def make_tp_setattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + setattr_fn = w_type.lookup('__setattr__') + delattr_fn = w_type.lookup('__delattr__') + if setattr_fn is None or delattr_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, + error=-1) + @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) + def slot_tp_setattro(space, w_self, w_name, w_value): + if w_value is not None: + space.call_function(setattr_fn, w_self, w_name, w_value) + else: + space.call_function(delattr_fn, w_self, w_name) + return 0 + return slot_tp_setattro + + at slot_factory('tp_getattro') +def make_tp_getattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + getattr_fn = w_type.lookup('__getattribute__') + if getattr_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) + def slot_tp_getattro(space, w_self, w_name): + return space.call_function(getattr_fn, w_self, w_name) + return slot_tp_getattro + + at slot_factory('tp_call') +def make_tp_call(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + call_fn = w_type.lookup('__call__') + if call_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(call_fn, args) + return slot_tp_call + + at slot_factory('tp_iternext') +def make_tp_iternext(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + iternext_fn = w_type.lookup('__next__') + if iternext_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_iternext(space, w_self): + try: + return space.call_function(iternext_fn, w_self) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + return None + return slot_tp_iternext + + at slot_factory('tp_init') +def make_tp_init(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + init_fn = w_type.lookup('__init__') + if init_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_init(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + space.call_args(init_fn, args) + return 0 + return slot_tp_init + + at slot_factory('tp_new') +def make_tp_new(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + new_fn = w_type.lookup('__new__') + if new_fn is None: + return + + @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_new(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(space.get(new_fn, w_self), args) + return slot_tp_new + + at slot_factory('tp_as_buffer.c_bf_getbuffer') +def make_bf_getbuffer(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + buff_fn = w_type.lookup('__buffer__') + if buff_fn is not None: + return slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + return slot_from_buffer_w(space, typedef) + else: + return None + + at slot_factory('tp_descr_get') +def make_tp_descr_get(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + get_fn = w_type.lookup('__get__') + if get_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_get(space, w_self, w_obj, w_value): + if w_obj is None: + w_obj = space.w_None + return space.call_function(get_fn, w_self, w_obj, w_value) + return slot_tp_descr_get + + at slot_factory('tp_descr_set') +def make_tp_descr_set(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + set_fn = w_type.lookup('__set__') + delete_fn = w_type.lookup('__delete__') + if set_fn is None and delete_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_set(space, w_self, w_obj, w_value): + if w_value is not None: + if set_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __set__", typedef.name) + space.call_function(set_fn, w_self, w_obj, w_value) + else: + if delete_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __delete__", typedef.name) + space.call_function(delete_fn, w_self, w_obj) + return 0 + return slot_tp_descr_set + def slot_from___buffer__(space, typedef, buff_fn): name = 'bf_getbuffer' @@ -719,7 +760,7 @@ return 0 return buff_w -def slot_from_buffer_w(space, typedef, buff_fn): +def slot_from_buffer_w(space, typedef): name = 'bf_getbuffer' @slot_function([PyObject, Py_bufferP, rffi.INT_real], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -228,7 +228,7 @@ SLOTS = {} @specialize.memo() -def get_slot_tp_function(space, typedef, name): +def get_slot_tp_function(space, typedef, name, method_name): """Return a description of the slot C function to use for the built-in type for 'typedef'. The 'name' is the slot name. This is a memo function that, after translation, returns one of a built-in finite set. @@ -237,7 +237,7 @@ try: return SLOTS[key] except KeyError: - slot_func = build_slot_tp_function(space, typedef, name) + slot_func = build_slot_tp_function(space, typedef, name, method_name) api_func = slot_func.api_func if slot_func else None SLOTS[key] = api_func return api_func @@ -281,7 +281,7 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_apifunc = get_slot_tp_function(space, typedef, slot_name) + slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name) if not slot_apifunc: warn_missing_slot(space, method_name, slot_name, w_type) continue From pypy.commits at gmail.com Tue Feb 20 15:34:03 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Feb 2018 12:34:03 -0800 (PST) Subject: [pypy-commit] pypy throw-away: investigate sdk env variables looking for WindowsSdkVerBinPath? Message-ID: <5a8c863b.949c1c0a.c0db0.dd0a@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93849:f426d6f2efde Date: 2018-02-20 22:33 -0500 http://bitbucket.org/pypy/pypy/changeset/f426d6f2efde/ Log: investigate sdk env variables looking for WindowsSdkVerBinPath? diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -83,7 +83,6 @@ stderr=subprocess.PIPE) stdout, stderr = popen.communicate() - print 'running', vcvars, "\nstderr '%s'" % stderr, '\nstdout' if popen.wait() != 0: return None except: @@ -93,12 +92,14 @@ vcdict = {} for line in stdout.split("\n"): if '=' not in line: - print line continue key, value = line.split('=', 1) vcdict[key] = value env = {} + log.msg('got msvc environ') for key, value in vcdict.items(): + if 'sdk' in key.lower(): + log.msg('%s=%s' %(key, value)) if key.upper() in ['PATH', 'INCLUDE', 'LIB']: env[key.upper()] = value log.msg("Updated environment with vsver %d, using x64 %s" % (vsver, x64flag,)) @@ -279,16 +280,7 @@ mfid = 2 out_arg = '-outputresource:%s;%s' % (exe_name, mfid) args = ['-nologo', '-manifest', str(temp_manifest), out_arg] - try: - self._execute_c_compiler('mt.exe', args, exe_name) - except EnvironmentError: - paths = self.c_environ['PATH'].split(';') - print 'paths, looking for mt.exe' - for p in paths: - print p - if 'Kit' in p: - print ' \n'.join(os.listdir(p)) - raise + self._execute_c_compiler('mt.exe', args, exe_name) return exe_name From pypy.commits at gmail.com Tue Feb 20 23:10:13 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Feb 2018 20:10:13 -0800 (PST) Subject: [pypy-commit] pypy default: fix translation Message-ID: <5a8cf125.c21b1c0a.d376c.8c7d@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93851:fc072d7375a3 Date: 2018-02-21 04:09 +0000 http://bitbucket.org/pypy/pypy/changeset/fc072d7375a3/ Log: fix translation diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -442,9 +442,9 @@ return SLOTS[key] except KeyError: slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) - llfunc = llslot(space, slot_func) if slot_func else None - SLOTS[key] = llfunc - return llfunc + api_func = slot_func.api_func if slot_func else None + SLOTS[key] = api_func + return api_func def make_unary_slot(space, typedef, name, attr): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -265,10 +265,11 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_llfunc = get_slot_tp_function(space, typedef, slot_name, method_name) - if not slot_llfunc: + slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name) + if not slot_apifunc: warn_missing_slot(space, method_name, slot_name, w_type) continue + slot_llfunc = slot_apifunc.get_llhelper(space) fill_slot(space, pto, w_type, slot_names, slot_llfunc) @specialize.arg(3) From pypy.commits at gmail.com Wed Feb 21 12:18:31 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Feb 2018 09:18:31 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: temporary debug prints to figure out why heap knowledge gets lost Message-ID: <5a8da9e7.02c21c0a.7cff0.09cb@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93852:3f9f732b42d0 Date: 2018-02-20 11:40 +0100 http://bitbucket.org/pypy/pypy/changeset/3f9f732b42d0/ Log: temporary debug prints to figure out why heap knowledge gets lost diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -3,6 +3,8 @@ from rpython.jit.metainterp import resumecode +from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop, + debug_print) # adds the following sections at the end of the resume code: # @@ -59,7 +61,6 @@ for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -107,6 +108,8 @@ numb_state.append_int(0) def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + debug_start("jit-bridge-deserialize") + reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -116,6 +119,7 @@ reader.jump(startcount - 1) # class knowledge + debug_print("# class knowledge") bitfield = 0 mask = 0 for i, box in enumerate(liveboxes): @@ -129,9 +133,14 @@ if class_known: cls = optimizer.cpu.ts.cls_of_box(frontend_boxes[i]) optimizer.make_constant_class(box, cls) + debug_print("class known", i) + else: + debug_print("class NOT known", i) # heap knowledge + debug_print("# heap knowledge") if not optimizer.optheap: + debug_print("# no optheap!") return length = reader.next_item() result_struct = [] @@ -143,6 +152,15 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_struct.append((box1, descr, box2)) + + attrname = descr.repr_of_descr() + try: + i1 = liveboxes.index(box1) + i2 = liveboxes.index(box2) + except ValueError: + debug_print("getfield: some unknown box with field", attrname) + else: + debug_print("getfield:", i1, i2, attrname) length = reader.next_item() result_array = [] for i in range(length): @@ -155,3 +173,5 @@ box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) optimizer.optheap.deserialize_optheap(result_struct, result_array) + + debug_stop("jit-bridge-deserialize") From pypy.commits at gmail.com Wed Feb 21 12:18:34 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Feb 2018 09:18:34 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: simplify optimizer interface to guard-compat. also add more debug prints Message-ID: <5a8da9ea.0e781c0a.e4feb.46f6@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93853:4db603258cdc Date: 2018-02-20 12:13 +0100 http://bitbucket.org/pypy/pypy/changeset/4db603258cdc/ Log: simplify optimizer interface to guard-compat. also add more debug prints diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -83,8 +83,12 @@ cond.activate_secondary(ref, loop_token) return True - def prepare_const_arg_call(self, op, optimizer): - copied_op, cond = self._prepare_const_arg_call(op, optimizer) + def replace_with_const_result_if_possible(self, op, optimizer): + """ Main entry point for optimizeopt: check if op can be replaced with + a constant result and stored in the conditions instead of leaving it in + the trace. Returns None if that is not possible. """ + copied_op, cond, failure_reason = self._prepare_const_arg_call( + op, optimizer) if copied_op: result = optimizer._can_optimize_call_pure(copied_op) if result is None: @@ -93,9 +97,23 @@ result = do_call( optimizer.cpu, copied_op.getarglist(), copied_op.getdescr()) - return copied_op, cond, result - else: - return None, None, None + recorded = self.record_condition( + cond, result, optimizer) + if recorded: + return result + else: + failure_reason = "self is frozen" + debug_start("jit-guard-compatible") + if have_debug_prints(): + try: + addr = op.getarg(0).getaddr() + except Exception: + funcname = "?" + else: + funcname = optimizer.metainterp_sd.get_name_from_address(addr) + debug_print("failed to optimize call:", funcname, failure_reason) + debug_stop("jit-guard-compatible") + return None def _prepare_const_arg_call(self, op, optimizer): from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr @@ -112,15 +130,16 @@ copied_op = op.copy() copied_op.setarg(1, self.known_valid) if op.numargs() == 2: - return copied_op, PureCallCondition(op, optimizer) + return copied_op, PureCallCondition(op, optimizer), "" arg2 = copied_op.getarg(2) if arg2.is_constant(): # already a constant, can just use PureCallCondition if last_nonconst_index != -1: - return None, None # a non-constant argument, can't optimize - return copied_op, PureCallCondition(op, optimizer) + # a non-constant argument, can't optimize + return None, None, "nonconstant arg " + str(last_nonconst_index) + return copied_op, PureCallCondition(op, optimizer), "" if last_nonconst_index != 2: - return None, None + return None, None, "nonconstant arg " + str(last_nonconst_index) # really simple-minded pattern matching # the order of things is like this: @@ -132,20 +151,20 @@ # make it possible for the GUARD_COMPATIBLE to still remove the call, # even though the second argument is not constant if arg2.getopnum() not in (rop.GETFIELD_GC_R, rop.GETFIELD_GC_I, rop.GETFIELD_GC_F): - return None, None + return None, None, "arg2 not a getfield" if not self.last_quasi_immut_field_op: - return None, None + return None, None, "no quasiimut op known" qmutdescr = self.last_quasi_immut_field_op.getdescr() assert isinstance(qmutdescr, QuasiImmutDescr) fielddescr = qmutdescr.fielddescr # XXX same_arg = self.last_quasi_immut_field_op.getarg(0) is arg2.getarg(0) if arg2.getdescr() is not fielddescr or not same_arg: - return None, None + return None, None, "fielddescr doesn't match" if not qmutdescr.is_still_valid_for(self.known_valid): - return None, None + return None, None, "qmutdescr invalid" copied_op.setarg(2, qmutdescr.constantfieldbox) return copied_op, QuasiimmutGetfieldAndPureCallCondition( - op, qmutdescr, optimizer) + op, qmutdescr, optimizer), "" def emit_conditions(self, op, short, optimizer): """ re-emit the conditions about variable op into the short preamble diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -166,15 +166,12 @@ ccond = info._compatibility_conditions if ccond: # it's subject to guard_compatible - copied_op, cond, result = ccond.prepare_const_arg_call( - op, self.optimizer) - if copied_op: - recorded = ccond.record_condition( - cond, result, self.optimizer) - if recorded: - self.make_constant(op, result) - self.last_emitted_operation = REMOVED - return + result = ccond.replace_with_const_result_if_possible( + op, self.optimizer) + if result: + self.make_constant(op, result) + self.last_emitted_operation = REMOVED + return # Step 1: check if all arguments are constant for i in range(start_index, op.numargs()): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -531,7 +531,7 @@ op = ResOperation( rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr)], descr=self.plaincalldescr) - copied_op, cond, result = ccond.prepare_const_arg_call( + copied_op, cond, reason = ccond._prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) @@ -545,7 +545,7 @@ rop.CALL_PURE_I, [ConstInt(123), ConstPtr(self.quasiptr), getfield_op], descr=self.nonwritedescr) - copied_op, cond, result = ccond.prepare_const_arg_call( + copied_op, cond, reason = ccond._prepare_const_arg_call( op, optimizer) ccond.record_condition(cond, ConstInt(5), optimizer) value = info.PtrInfo() From pypy.commits at gmail.com Wed Feb 21 12:18:36 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Feb 2018 09:18:36 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: log the guard address too Message-ID: <5a8da9ec.8583df0a.e7cbe.c23c@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93854:25fb71c78772 Date: 2018-02-21 17:19 +0100 http://bitbucket.org/pypy/pypy/changeset/25fb71c78772/ Log: log the guard address too diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -108,9 +108,14 @@ numb_state.append_int(0) def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.rlib.objectmodel import compute_unique_id + from rpython.rlib.rarithmetic import r_uint + debug_start("jit-bridge-deserialize") reader = resumecode.Reader(resumestorage.rd_numb) + hash = r_uint(compute_unique_id(resumestorage)) + debug_print("resuming from Guard0x%x" % hash) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd From pypy.commits at gmail.com Wed Feb 21 12:18:38 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Feb 2018 09:18:38 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: test and fix: deal with quasiimmut correctly Message-ID: <5a8da9ee.42addf0a.f7dfd.5d5d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93855:5b83d5703851 Date: 2018-02-21 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/5b83d5703851/ Log: test and fix: deal with quasiimmut correctly diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -62,8 +62,12 @@ self.conditions.append(cond) return True - def register_quasi_immut_field(self, op): - self.last_quasi_immut_field_op = op + def register_quasi_immut_field(self, op, optimizer): + from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr + assert optimizer.ensure_ptr_info_arg0(op)._compatibility_conditions is self + descr = op.getdescr() + assert isinstance(descr, QuasiImmutDescr) + self.last_quasi_immut_field_descr = descr def check_compat(self, cpu, ref): for i, cond in enumerate(self.conditions): @@ -152,13 +156,12 @@ # even though the second argument is not constant if arg2.getopnum() not in (rop.GETFIELD_GC_R, rop.GETFIELD_GC_I, rop.GETFIELD_GC_F): return None, None, "arg2 not a getfield" - if not self.last_quasi_immut_field_op: + if not self.last_quasi_immut_field_descr: return None, None, "no quasiimut op known" - qmutdescr = self.last_quasi_immut_field_op.getdescr() + qmutdescr = self.last_quasi_immut_field_descr assert isinstance(qmutdescr, QuasiImmutDescr) fielddescr = qmutdescr.fielddescr # XXX - same_arg = self.last_quasi_immut_field_op.getarg(0) is arg2.getarg(0) - if arg2.getdescr() is not fielddescr or not same_arg: + if arg2.getdescr() is not fielddescr: return None, None, "fielddescr doesn't match" if not qmutdescr.is_still_valid_for(self.known_valid): return None, None, "qmutdescr invalid" diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -677,7 +677,7 @@ # the getfield_gc on the object, since it's not constant. # However, if the quasi-immutable field is passed to a pure # function call, we can treat it as constant then - ccond.register_quasi_immut_field(op) + ccond.register_quasi_immut_field(op, self.optimizer) # don't remove the guard_not_invalidated, the guard_compatible # needs it self._remove_guard_not_invalidated = False diff --git a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py --- a/rpython/jit/metainterp/optimizeopt/test/test_compatible.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_compatible.py @@ -281,6 +281,55 @@ } self.optimize_loop(ops, expected, call_pure_results) + def test_quasiimmut_bug(self): + # bug that happened because we hade quasiimmut_field on two different + # boxes (that turn out to be the same after other optimizations) + ops = """ + [p1] + guard_compatible(p1, ConstPtr(quasiptr)) [] + quasiimmut_field(p1, descr=quasiimmutdescr) + guard_not_invalidated() [] + i0 = getfield_gc_i(p1, descr=quasifielddescr) + i1 = call_pure_i(123, p1, i0, descr=nonwritedescr) + + # get an alias p2 to p1 + p3 = new_with_vtable(descr=nodesize) + setfield_gc(p3, p1, descr=nextdescr) + p2 = getfield_gc_r(p3, descr=nextdescr) + + # a condition via the alias + guard_compatible(p2, ConstPtr(quasiptr)) [] + quasiimmut_field(p2, descr=quasiimmutdescr) + guard_not_invalidated() [] + i3 = getfield_gc_i(p2, descr=quasifielddescr) + i4 = call_pure_i(123, p2, i3, descr=nonwritedescr) + + # and now the original box + i5 = call_pure_i(123, p1, i0, descr=nonwritedescr) + escape_n(i1) + escape_n(i4) + escape_n(i5) + jump(p1) + """ + expected = """ + [p1] + guard_compatible(p1, ConstPtr(quasiptr)) [] + guard_not_invalidated() [] + i0 = getfield_gc_i(p1, descr=quasifielddescr) # will be removed by the backend + escape_n(5) + escape_n(5) + escape_n(5) + jump(p1) + """ + call_pure_results = { + (ConstInt(123), ConstPtr(self.quasiptr), ConstInt(-4247)): ConstInt(5), + } + self.optimize_loop(ops, expected, call_pure_results) + descr = self.loop.operations[1].getdescr() + assert descr._compatibility_conditions is not None + assert descr._compatibility_conditions.known_valid.same_constant(ConstPtr(self.quasiptr)) + assert len(descr._compatibility_conditions.conditions) == 1 + class TestCompatibleUnroll(BaseTestWithUnroll, LLtypeMixin): From pypy.commits at gmail.com Wed Feb 21 15:16:13 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 12:16:13 -0800 (PST) Subject: [pypy-commit] pypy throw-away: try to fix missing path name on buildbot Message-ID: <5a8dd38d.2196df0a.a6111.4da0@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93856:2956c36a5584 Date: 2018-02-21 22:15 -0500 http://bitbucket.org/pypy/pypy/changeset/2956c36a5584/ Log: try to fix missing path name on buildbot diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -73,7 +73,7 @@ if not os.path.exists(vcvars): # even msdn does not know which to run # see https://msdn.microsoft.com/en-us/library/1700bbwd(v=vs.90).aspx - # wich names both + # which names both vcvars = os.path.join(toolsdir, 'vcvars32.bat') import subprocess @@ -102,6 +102,18 @@ log.msg('%s=%s' %(key, value)) if key.upper() in ['PATH', 'INCLUDE', 'LIB']: env[key.upper()] = value + if not _find_executable('mt.exe', env['PATH']): + # For some reason the sdk bin path is missing? + # put it together from some other env variables that happened to exist + # on the buildbot where this occurred + if 'WindowsSDKVersion' in vcdict and 'WindowsSdkDir' in vcdict: + binpath = vcdict['WindowsSdkDir'] + '\\bin' + vcdict['WindowsSDKVersion'] + 'x86' + env['PATH'] += ';' + binpath + if not _find_executable('mt.exe', env['PATH']): + log.msg('Could not find mt.exe on path=%s' % env['PATH']) + log.msg('Running vsver %s set this env' % vsver) + for key, value in vcdict.items(): + log.msg('%s=%s' %(key, value)) log.msg("Updated environment with vsver %d, using x64 %s" % (vsver, x64flag,)) return env From pypy.commits at gmail.com Wed Feb 21 15:22:38 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 12:22:38 -0800 (PST) Subject: [pypy-commit] pypy throw-away: typo in path Message-ID: <5a8dd50e.c185df0a.d8285.48f2@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93857:428b5c69b2b3 Date: 2018-02-21 22:21 -0500 http://bitbucket.org/pypy/pypy/changeset/428b5c69b2b3/ Log: typo in path diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -96,10 +96,7 @@ key, value = line.split('=', 1) vcdict[key] = value env = {} - log.msg('got msvc environ') for key, value in vcdict.items(): - if 'sdk' in key.lower(): - log.msg('%s=%s' %(key, value)) if key.upper() in ['PATH', 'INCLUDE', 'LIB']: env[key.upper()] = value if not _find_executable('mt.exe', env['PATH']): @@ -107,7 +104,7 @@ # put it together from some other env variables that happened to exist # on the buildbot where this occurred if 'WindowsSDKVersion' in vcdict and 'WindowsSdkDir' in vcdict: - binpath = vcdict['WindowsSdkDir'] + '\\bin' + vcdict['WindowsSDKVersion'] + 'x86' + binpath = vcdict['WindowsSdkDir'] + '\\bin\\' + vcdict['WindowsSDKVersion'] + 'x86' env['PATH'] += ';' + binpath if not _find_executable('mt.exe', env['PATH']): log.msg('Could not find mt.exe on path=%s' % env['PATH']) From pypy.commits at gmail.com Wed Feb 21 16:56:32 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 13:56:32 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5a8deb10.42badf0a.b2257.1bd3@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93858:7019087f6069 Date: 2018-02-21 23:50 -0500 http://bitbucket.org/pypy/pypy/changeset/7019087f6069/ Log: merge default into py3.5 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -430,9 +430,9 @@ return SLOTS[key] except KeyError: slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) - llfunc = llslot(space, slot_func) if slot_func else None - SLOTS[key] = llfunc - return llfunc + api_func = slot_func.api_func if slot_func else None + SLOTS[key] = api_func + return api_func def make_unary_slot(space, typedef, name, attr): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -265,10 +265,11 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_llfunc = get_slot_tp_function(space, typedef, slot_name, method_name) - if not slot_llfunc: + slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name) + if not slot_apifunc: warn_missing_slot(space, method_name, slot_name, w_type) continue + slot_llfunc = slot_apifunc.get_llhelper(space) fill_slot(space, pto, w_type, slot_names, slot_llfunc) @specialize.arg(3) From pypy.commits at gmail.com Wed Feb 21 16:56:36 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 13:56:36 -0800 (PST) Subject: [pypy-commit] pypy py3.5: stdint.h exists for msver 1900 (vs 2017) Message-ID: <5a8deb14.4d051c0a.735f5.9712@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93860:a745d2c294be Date: 2018-02-21 23:55 -0500 http://bitbucket.org/pypy/pypy/changeset/a745d2c294be/ Log: stdint.h exists for msver 1900 (vs 2017) diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h --- a/lib_pypy/_libmpdec/vccompat.h +++ b/lib_pypy/_libmpdec/vccompat.h @@ -32,7 +32,9 @@ /* Visual C fixes: no stdint.h, no snprintf ... */ #ifdef _MSC_VER + #if _MSC_VER < 1900 #include "vcstdint.h" + #endif #undef inline #define inline __inline #undef random From pypy.commits at gmail.com Wed Feb 21 16:56:34 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 13:56:34 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hack to fix missing path component on a buildbot with vs2017 installed Message-ID: <5a8deb12.4f5b1c0a.28fa6.932c@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93859:2b8826f6536c Date: 2018-02-21 23:53 -0500 http://bitbucket.org/pypy/pypy/changeset/2b8826f6536c/ Log: hack to fix missing path component on a buildbot with vs2017 installed diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -73,7 +73,7 @@ if not os.path.exists(vcvars): # even msdn does not know which to run # see https://msdn.microsoft.com/en-us/library/1700bbwd(v=vs.90).aspx - # wich names both + # which names both vcvars = os.path.join(toolsdir, 'vcvars32.bat') import subprocess @@ -99,6 +99,18 @@ for key, value in vcdict.items(): if key.upper() in ['PATH', 'INCLUDE', 'LIB']: env[key.upper()] = value + if not _find_executable('mt.exe', env['PATH']): + # For some reason the sdk bin path is missing? + # put it together from some other env variables that happened to exist + # on the buildbot where this occurred + if 'WindowsSDKVersion' in vcdict and 'WindowsSdkDir' in vcdict: + binpath = vcdict['WindowsSdkDir'] + '\\bin\\' + vcdict['WindowsSDKVersion'] + 'x86' + env['PATH'] += ';' + binpath + if not _find_executable('mt.exe', env['PATH']): + log.msg('Could not find mt.exe on path=%s' % env['PATH']) + log.msg('Running vsver %s set this env' % vsver) + for key, value in vcdict.items(): + log.msg('%s=%s' %(key, value)) log.msg("Updated environment with vsver %d, using x64 %s" % (vsver, x64flag,)) return env From pypy.commits at gmail.com Wed Feb 21 17:43:25 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 14:43:25 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hack to add externals to INCLUDE, LIB, BIN on windows Message-ID: <5a8df60d.0ea7df0a.149c7.e596@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93862:759ae332758e Date: 2018-02-22 00:43 +0200 http://bitbucket.org/pypy/pypy/changeset/759ae332758e/ Log: hack to add externals to INCLUDE, LIB, BIN on windows diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -210,6 +210,10 @@ if key == '_ssl' and sys.platform == 'darwin': # needed for our roots patch env['LDFLAGS'] += ' -framework CoreFoundation -framework Security' + elif sys.platform == 'win32': + env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '') + env['LIB'] = r'..\externals\lib;' + env.get('LIB', '') + env['PATH'] = r'..\externals\bin;' + env.get('PATH', '') try: status, stdout, stderr = run_subprocess(str(pypy_c), args, From pypy.commits at gmail.com Wed Feb 21 17:43:23 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 14:43:23 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix decimal Message-ID: <5a8df60b.8f98df0a.ede96.6882@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93861:e7ca5c644d47 Date: 2018-02-22 00:42 +0200 http://bitbucket.org/pypy/pypy/changeset/e7ca5c644d47/ Log: fix decimal diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h --- a/lib_pypy/_libmpdec/vccompat.h +++ b/lib_pypy/_libmpdec/vccompat.h @@ -34,6 +34,8 @@ #ifdef _MSC_VER #if _MSC_VER < 1900 #include "vcstdint.h" + #else + #include "stdint.h" #endif #undef inline #define inline __inline From pypy.commits at gmail.com Wed Feb 21 17:53:10 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 14:53:10 -0800 (PST) Subject: [pypy-commit] pypy py3.5: vcvars no longer sets libpath, it is unused anyway Message-ID: <5a8df856.94a1df0a.2afa8.bb48@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93864:87cbb9df283a Date: 2018-02-22 00:52 -0500 http://bitbucket.org/pypy/pypy/changeset/87cbb9df283a/ Log: vcvars no longer sets libpath, it is unused anyway diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -255,7 +255,7 @@ """Launch vcvarsall.bat and read the settings from its environment """ vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) + interesting = set(("include", "lib", "path")) result = {} if vcvarsall is None: From pypy.commits at gmail.com Wed Feb 21 17:53:08 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Feb 2018 14:53:08 -0800 (PST) Subject: [pypy-commit] pypy throw-away: close throw-away branch Message-ID: <5a8df854.ad86df0a.9e551.62eb@mx.google.com> Author: Matti Picus Branch: throw-away Changeset: r93863:b579b2e8a71e Date: 2018-02-21 23:57 -0500 http://bitbucket.org/pypy/pypy/changeset/b579b2e8a71e/ Log: close throw-away branch From pypy.commits at gmail.com Thu Feb 22 01:46:35 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 21 Feb 2018 22:46:35 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #358 Message-ID: <5a8e674b.c99adf0a.77312.8e59@mx.google.com> Author: Armin Rigo Branch: Changeset: r3102:7b46e415e208 Date: 2018-02-22 07:46 +0100 http://bitbucket.org/cffi/cffi/changeset/7b46e415e208/ Log: Issue #358 Fix following recent changes to CPython 2.7 diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ From pypy.commits at gmail.com Thu Feb 22 01:46:38 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 21 Feb 2018 22:46:38 -0800 (PST) Subject: [pypy-commit] cffi default: Document the fixes since 1.11.4 Message-ID: <5a8e674e.c5061c0a.5071e.8170@mx.google.com> Author: Armin Rigo Branch: Changeset: r3103:3c4397c47569 Date: 2018-02-22 07:46 +0100 http://bitbucket.org/cffi/cffi/changeset/3c4397c47569/ Log: Document the fixes since 1.11.4 diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -3,13 +3,30 @@ ====================== -v1.11.5 +v1.11.6 ======= * `Issue #357`_: fix ``ffi.emit_python_code()`` which generated a buggy Python file if you are using a ``struct`` with an anonymous ``union`` field or vice-versa. +* Windows: ``ffi.dlopen()`` should now handle unicode filenames. + +* Implemented ``ffi.dlclose()`` for the in-line case (it used to be + present only in the out-of-line case). + +* Fixed a corner case for ``setup.py install --record=xx --root=yy`` + with an out-of-line ABI module. + +* More hacks on Windows for running CFFI's own ``setup.py``. + +* `Issue #358`_: in embedding, to protect against Python initialization + from several threads in parallel, we have to use a spin-lock. On + CPython 3 it is worse because it might spin-lock for a long time + (execution of ``Py_InitializeEx()``). Sadly, recent changes to + CPython make that solution needed on CPython 2 too. + +.. _`Issue #358`: https://bitbucket.org/cffi/cffi/issues/358/ .. _`Issue #357`: https://bitbucket.org/cffi/cffi/issues/357/ From pypy.commits at gmail.com Thu Feb 22 02:05:17 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 21 Feb 2018 23:05:17 -0800 (PST) Subject: [pypy-commit] pypy default: Issue #2755 Message-ID: <5a8e6bad.92aa1c0a.11b50.24f5@mx.google.com> Author: Armin Rigo Branch: Changeset: r93865:0b2256471902 Date: 2018-02-22 08:02 +0100 http://bitbucket.org/pypy/pypy/changeset/0b2256471902/ Log: Issue #2755 Accept unicode filenames in dbm.open() diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -157,7 +157,14 @@ def open(filename, flag='r', mode=0666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") openflag = 0 diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/pypy/module/test_lib_pypy/test_dbm_extra.py @@ -1,5 +1,5 @@ from __future__ import absolute_import -import py +import py, os from rpython.tool.udir import udir try: from lib_pypy import dbm @@ -74,3 +74,8 @@ assert 'key_with_empty_value' in d assert d['key_with_empty_value'] == '' d.close() + +def test_unicode_filename(): + path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' + d = dbm.open(path, 'c') + d.close() From pypy.commits at gmail.com Thu Feb 22 02:05:20 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 21 Feb 2018 23:05:20 -0800 (PST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5a8e6bb0.16b7df0a.bef52.6250@mx.google.com> Author: Armin Rigo Branch: Changeset: r93866:8f992014885b Date: 2018-02-22 08:04 +0100 http://bitbucket.org/pypy/pypy/changeset/8f992014885b/ Log: merge heads diff too long, truncating to 2000 out of 4036 lines diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + + at pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -40,3 +40,7 @@ .. branch: memory-accounting Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,5 +1,6 @@ import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be) @@ -26,7 +27,10 @@ got = decode_utf8(space, "\xed\xa0\x80\xed\xb0\x80") assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] @pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) def test_utf32_surrogates(unich): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'get_hidden_tb' : 'interp_magic.get_hidden_tb', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -107,14 +107,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - def get_console_cp(space): from rpython.rlib import rwin32 # Windows only return space.newtuple([ diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -538,11 +538,16 @@ assert '\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 def test_badhandler(self): import codecs diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -96,25 +96,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -31,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -128,6 +133,7 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + attach=type_attach, dealloc=date_dealloc, ) @@ -138,8 +144,10 @@ def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -6,7 +6,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, - Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) + Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, @@ -143,7 +143,6 @@ def wrap_inquirypred(space, w_self, w_args, func): func_inquiry = rffi.cast(inquiry, func) check_num_args(space, w_args, 0) - args_w = space.fixedview(w_args) res = generic_cpy_call(space, func_inquiry, w_self) res = rffi.cast(lltype.Signed, res) if res == -1: @@ -423,285 +422,331 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): + +SLOTS = {} + at specialize.memo() +def get_slot_tp_function(space, typedef, name, method_name): + """Return a description of the slot C function to use for the built-in + type for 'typedef'. The 'name' is the slot name. This is a memo + function that, after translation, returns one of a built-in finite set. + """ + key = (typedef, name) + try: + return SLOTS[key] + except KeyError: + slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) + api_func = slot_func.api_func if slot_func else None + SLOTS[key] = api_func + return api_func + + +def make_unary_slot(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - - handled = False - # unary functions - for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_as_number.c_nb_hex', '__hex__'), - ('tp_as_number.c_nb_oct', '__oct__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('next') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set - else: - # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce - # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length - # richcmpfunc(s) + slot_fn = w_type.lookup(attr) + if slot_fn is None: return + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) return slot_func +UNARY_SLOTS = [ + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + + at slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + + at slot_factory('tp_as_mapping.c_mp_ass_subscript') +def make_sq_set_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, w_arg1, w_arg2) + else: + space.call_function(slot_del, w_self, w_arg1) + return 0 + return slot_func + + + at slot_factory('tp_as_sequence.c_sq_ass_item') +def make_sq_ass_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) + else: + space.call_function(slot_del, w_self, space.newint(arg1)) + return 0 + return slot_func + +def make_tp_setattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + setattr_fn = w_type.lookup('__setattr__') + delattr_fn = w_type.lookup('__delattr__') + if setattr_fn is None or delattr_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, + error=-1) + @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) + def slot_tp_setattro(space, w_self, w_name, w_value): + if w_value is not None: + space.call_function(setattr_fn, w_self, w_name, w_value) + else: + space.call_function(delattr_fn, w_self, w_name) + return 0 + return slot_tp_setattro + + at slot_factory('tp_getattro') +def make_tp_getattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + getattr_fn = w_type.lookup('__getattribute__') + if getattr_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) + def slot_tp_getattro(space, w_self, w_name): + return space.call_function(getattr_fn, w_self, w_name) + return slot_tp_getattro + + at slot_factory('tp_call') +def make_tp_call(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + call_fn = w_type.lookup('__call__') + if call_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(call_fn, args) + return slot_tp_call + + at slot_factory('tp_iternext') +def make_tp_iternext(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + iternext_fn = w_type.lookup('next') + if iternext_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_iternext(space, w_self): + try: + return space.call_function(iternext_fn, w_self) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + return None + return slot_tp_iternext + + at slot_factory('tp_init') +def make_tp_init(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + init_fn = w_type.lookup('__init__') + if init_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_init(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + space.call_args(init_fn, args) + return 0 + return slot_tp_init + + at slot_factory('tp_new') +def make_tp_new(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + new_fn = w_type.lookup('__new__') + if new_fn is None: + return + + @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_new(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(space.get(new_fn, w_self), args) + return slot_tp_new + + at slot_factory('tp_as_buffer.c_bf_getbuffer') +def make_bf_getbuffer(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + buff_fn = w_type.lookup('__buffer__') + if buff_fn is not None: + return slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + return slot_from_buffer_w(space, typedef) + else: + return None + + at slot_factory('tp_descr_get') +def make_tp_descr_get(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + get_fn = w_type.lookup('__get__') + if get_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_get(space, w_self, w_obj, w_value): + if w_obj is None: + w_obj = space.w_None + return space.call_function(get_fn, w_self, w_obj, w_value) + return slot_tp_descr_get + + at slot_factory('tp_descr_set') +def make_tp_descr_set(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + set_fn = w_type.lookup('__set__') + delete_fn = w_type.lookup('__delete__') + if set_fn is None and delete_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_set(space, w_self, w_obj, w_value): + if w_value is not None: + if set_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __set__", typedef.name) + space.call_function(set_fn, w_self, w_obj, w_value) + else: + if delete_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __delete__", typedef.name) + space.call_function(delete_fn, w_self, w_obj) + return 0 + return slot_tp_descr_set + def slot_from___buffer__(space, typedef, buff_fn): name = 'bf_getbuffer' @@ -730,7 +775,7 @@ return 0 return buff_w -def slot_from_buffer_w(space, typedef, buff_fn): +def slot_from_buffer_w(space, typedef): name = 'bf_getbuffer' @slot_function([PyObject, Py_bufferP, rffi.INT_real], rffi.INT_real, error=-1) @@ -765,6 +810,28 @@ missing_wrapper.__name__ = name globals()[name] = missing_wrapper +def make_missing_slot(space, typedef, name, attr): + return None + +missing_builtin_slots = [ + 'tp_print', 'tp_compare', 'tp_getattr', 'tp_setattr', 'tp_setattro', + 'tp_richcompare', 'tp_del', 'tp_as_buffer.c_bf_getwritebuffer', + 'tp_as_number.c_nb_nonzero', 'tp_as_number.c_nb_coerce', + 'tp_as_number.c_nb_inplace_add', 'tp_as_number.c_nb_inplace_subtract', + 'tp_as_number.c_nb_inplace_multiply', 'tp_as_number.c_nb_inplace_divide', + 'tp_as_number.c_nb_inplace_remainder', 'tp_as_number.c_nb_inplace_power', + 'tp_as_number.c_nb_inplace_lshift', 'tp_as_number.c_nb_inplace_rshift', + 'tp_as_number.c_nb_inplace_and', 'tp_as_number.c_nb_inplace_xor', + 'tp_as_number.c_nb_inplace_or', + 'tp_as_number.c_nb_floor_divide', 'tp_as_number.c_nb_true_divide', + 'tp_as_number.c_nb_inplace_floor_divide', 'tp_as_number.c_nb_inplace_true_divide', + 'tp_as_sequence.c_sq_slice', 'tp_as_sequence.c_sq_ass_slice', + 'tp_as_sequence.c_sq_contains', + 'tp_as_buffer.c_bf_getreadbuffer', + ] +for name in missing_builtin_slots: + slot_factory(name)(make_missing_slot) + PyWrapperFlag_KEYWORDS = 1 diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -82,16 +82,6 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) - def test_tzinfo(self, space): - w_tzinfo = space.appexec( - [], """(): - from datetime import tzinfo - return tzinfo() - """) - assert PyTZInfo_Check(space, w_tzinfo) - assert PyTZInfo_CheckExact(space, w_tzinfo) - assert not PyTZInfo_Check(space, space.w_None) - class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -272,3 +262,81 @@ 2000, 6, 6, 6, 6, 6, 6) assert module.test_time_macros() == datetime.time(6, 6, 6, 6) assert module.test_delta_macros() == datetime.timedelta(6, 6, 6) + + def test_tzinfo(self): + module = self.import_extension('foo', [ + ("time_with_tzinfo", "METH_O", + """ PyDateTime_IMPORT; + return PyDateTimeAPI->Time_FromTime( + 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); + """), + ("datetime_with_tzinfo", "METH_O", + """ + PyObject * obj; + int tzrefcnt = args->ob_refcnt; + PyDateTime_IMPORT; + obj = PyDateTimeAPI->DateTime_FromDateAndTime( + 2000, 6, 6, 6, 6, 6, 6, args, + PyDateTimeAPI->DateTimeType); + if (!((PyDateTime_DateTime*)obj)->hastzinfo) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "missing tzinfo"); + return NULL; + } + if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not incremented"); + return NULL; + } + return obj; + + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time + # copied from datetime documentation + class GMT1(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + self.dst(dt) + def dst(self, dt): + return timedelta(0) + def tzname(self,dt): + return "GMT +1" + gmt1 = GMT1() + dt1 = module.time_with_tzinfo(gmt1) + assert dt1 == time(6, 6, 6, 6, gmt1) + assert '+01' in str(dt1) + assert module.datetime_with_tzinfo(gmt1) == datetime( + 2000, 6, 6, 6, 6, 6, 6, gmt1) + + def test_checks(self): + module = self.import_extension('foo', [ + ("checks", "METH_O", + """ PyDateTime_IMPORT; + return PyTuple_Pack(10, + PyBool_FromLong(PyDateTime_Check(args)), + PyBool_FromLong(PyDateTime_CheckExact(args)), + PyBool_FromLong(PyDate_Check(args)), + PyBool_FromLong(PyDate_CheckExact(args)), + PyBool_FromLong(PyTime_Check(args)), + PyBool_FromLong(PyTime_CheckExact(args)), + PyBool_FromLong(PyDelta_Check(args)), + PyBool_FromLong(PyDelta_CheckExact(args)), + PyBool_FromLong(PyTZInfo_Check(args)), + PyBool_FromLong(PyTZInfo_CheckExact(args)) + ); + """), + ], prologue='#include "datetime.h"\n') + from datetime import tzinfo, datetime, timedelta, time, date + o = date(1, 1, 1) + assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6 + o = time(1, 1, 1) + assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4 + o = timedelta(1, 1, 1) + assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2 + o = datetime(1, 1, 1) + assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) + o = tzinfo() + assert module.checks(o) == (False,) * 8 + (True,) * 2 + diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -31,7 +31,7 @@ PyObject, make_ref, from_ref, get_typedescr, make_typedescr, track_reference, decref, as_pyobj) from pypy.module.cpyext.slotdefs import ( - slotdefs_for_tp_slots, slotdefs_for_wrappers, build_slot_tp_function, + slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function, llslot) from pypy.module.cpyext.state import State from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne @@ -226,22 +226,6 @@ dict_w[name] = w_descr i += 1 -SLOTS = {} - at specialize.memo() -def get_slot_tp_function(space, typedef, name): - """Return a description of the slot C function to use for the built-in - type for 'typedef'. The 'name' is the slot name. This is a memo - function that, after translation, returns one of a built-in finite set. - """ - key = (typedef, name) - try: - return SLOTS[key] - except KeyError: - slot_func = build_slot_tp_function(space, typedef, name) - api_func = slot_func.api_func if slot_func else None - SLOTS[key] = api_func - return api_func - missing_slots={} def warn_missing_slot(space, method_name, slot_name, w_type): if not we_are_translated(): @@ -281,12 +265,12 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_apifunc = get_slot_tp_function(space, typedef, slot_name) + slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name) if not slot_apifunc: warn_missing_slot(space, method_name, slot_name, w_type) continue - slot_func_helper = slot_apifunc.get_llhelper(space) - fill_slot(space, pto, w_type, slot_names, slot_func_helper) + slot_llfunc = slot_apifunc.get_llhelper(space) + fill_slot(space, pto, w_type, slot_names, slot_llfunc) @specialize.arg(3) def fill_slot(space, pto, w_type, slot_names, slot_func_helper): @@ -332,7 +316,7 @@ def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented - hash_not_impl = PyObject_HashNotImplemented.api_func.get_llhelper(space) + hash_not_impl = llslot(space, PyObject_HashNotImplemented) for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -72,7 +72,7 @@ def __repr__(self): if self._s.total_memory_pressure != -1: - extra = "\nmemory pressure: %s" % self.total_memory_pressure + extra = "\n memory pressure: %s" % self.total_memory_pressure else: extra = "" return """Total memory consumed: @@ -109,5 +109,5 @@ self.memory_allocated_sum) -def get_stats(): - return GcStats(gc._get_stats()) +def get_stats(memory_pressure=False): + return GcStats(gc._get_stats(memory_pressure=memory_pressure)) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,7 +1,6 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import sys @@ -80,25 +79,30 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen + if osname == 'posix': # POSIX: we want to check the file descriptor when fdopen() is called, # not later when we read or write data. So we call fstat(), letting # it raise if fd is invalid. - _validate_fd = posix.fstat + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object + + Return an open file object connected to a file descriptor.""" + try: + posix.fstat(fd) + except OSError as e: + raise IOError(e.errno, e.message) + return _fdopen(fd, mode, buffering) + else: - _validate_fd = validate_fd + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object -# Capture file.fdopen at import time, as some code replaces -# __builtins__.file with a custom function. -_fdopen = file.fdopen - -def fdopen(fd, mode='r', buffering=-1): - """fdopen(fd [, mode='r' [, buffering]]) -> file_object - - Return an open file object connected to a file descriptor.""" - _validate_fd(fd) - return _fdopen(fd, mode, buffering) - + Return an open file object connected to a file descriptor.""" + return _fdopen(fd, mode, buffering) def tmpfile(): """Create a temporary file. diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -18,7 +18,7 @@ USEMODULES += ['fcntl'] else: # On windows, os.popen uses the subprocess module - USEMODULES += ['_rawffi', 'thread', 'signal'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) @@ -300,8 +300,13 @@ # There used to be code here to ensure that fcntl is not faked # but we can't do that cleanly any more - exc = raises(OSError, posix.fdopen, fd) - assert exc.value.errno == errno.EBADF + try: + fid = posix.fdopen(fd) + fid.read(10) + except IOError as e: + assert e.errno == errno.EBADF + else: + assert False, "using result of fdopen(fd) on closed file must raise" def test_fdopen_hackedbuiltins(self): "Same test, with __builtins__.file removed" @@ -331,8 +336,17 @@ path = self.path posix = self.posix fd = posix.open(path, posix.O_RDONLY) - exc = raises(OSError, posix.fdopen, fd, 'w') - assert str(exc.value) == "[Errno 22] Invalid argument" + # compatability issue - using Visual Studio 10 and above no + # longer raises on fid creation, only when _using_ fid + # win32 python2 raises IOError on flush(), win32 python3 raises OSError + try: + fid = posix.fdopen(fd, 'w') + fid.write('abc') + fid.flush() + except (OSError, IOError) as e: + assert e.errno in (9, 22) + else: + assert False, "expected OSError" posix.close(fd) # fd should not be closed def test_getcwd(self): diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/pypy/module/test_lib_pypy/pyrepl/test_functional.py deleted file mode 100644 --- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2000-2007 Michael Hudson-Doyle -# Maciek Fijalkowski -# License: MIT -# some functional tests, to see if this is really working - -import pytest -import sys - - -def pytest_funcarg__child(request): - try: - import pexpect - except ImportError: - pytest.skip("no pexpect module") - except SyntaxError: - pytest.skip('pexpect wont work on py3k') - child = pexpect.spawn(sys.executable, ['-S'], timeout=10) - child.logfile = sys.stdout - child.sendline('from pyrepl.python_reader import main') - child.sendline('main()') - return child - - -def test_basic(child): - child.sendline('a = 3') - child.sendline('a') - child.expect('3') diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -151,9 +151,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -187,20 +188,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -221,8 +236,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -296,7 +296,8 @@ def get_libc_name(): return rwin32.GetModuleFileName(get_libc_handle()) - assert "msvcr" in get_libc_name().lower(), \ + libc_name = get_libc_name().lower() + assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \ "Suspect msvcrt library: %s" % (get_libc_name(),) elif _MINGW: def get_libc_name(): diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,12 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3.3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include /* for _msize */ typedef struct { intptr_t osfhnd; @@ -95,6 +96,46 @@ errno = EBADF; return 0; } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: @@ -193,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): From pypy.commits at gmail.com Thu Feb 22 18:57:55 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 22 Feb 2018 15:57:55 -0800 (PST) Subject: [pypy-commit] pypy default: add externals/bin to PATH so tests pick up correct dlls Message-ID: <5a8f5903.4c7e1c0a.1451b.49f5@mx.google.com> Author: Matti Picus Branch: Changeset: r93872:ea67338ed935 Date: 2018-02-19 21:09 +0200 http://bitbucket.org/pypy/pypy/changeset/ea67338ed935/ Log: add externals/bin to PATH so tests pick up correct dlls (grafted from b0c75af579f53dcf071cef141b704c50e90fbaf8) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -535,7 +535,13 @@ # The buildbot runs get_externals before building def patch_env(env, externals = Platform.externals): #print 'adding %s to PATH, INCLUDE, LIB' % basepath - env['PATH'] = externals + r'\bin;' + env.get('PATH', '') + binpath = externals + r'\bin' + path = os.environ['PATH'] + if binpath not in path.split(';'): + path = binpath + ';' + path + # make sure externals is in current path for tests and translating + os.environ['PATH'] = path + env['PATH'] = binpath + ';' + env.get('PATH', '') env['INCLUDE'] = externals + r'\include;' + env.get('INCLUDE', '') env['LIB'] = externals + r'\lib;' + env.get('LIB', '') return None From pypy.commits at gmail.com Thu Feb 22 18:57:57 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 22 Feb 2018 15:57:57 -0800 (PST) Subject: [pypy-commit] pypy py3.5: skip time tests that cannot succeed on windows Message-ID: <5a8f5905.4c7e1c0a.1451b.49f9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93873:f99f8ea2e6bd Date: 2018-02-23 01:56 -0500 http://bitbucket.org/pypy/pypy/changeset/f99f8ea2e6bd/ Log: skip time tests that cannot succeed on windows diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py --- a/pypy/module/time/test/test_time.py +++ b/pypy/module/time/test/test_time.py @@ -107,7 +107,7 @@ t = time.time() assert time.localtime(t) == time.localtime(t) if os.name == 'nt': - raises(ValueError, time.localtime, -1) + raises(OSError, time.localtime, -1) else: time.localtime(-1) @@ -124,9 +124,15 @@ ltime = time.localtime() ltime = list(ltime) ltime[0] = -1 - time.mktime(tuple(ltime)) # Does not crash anymore + if os.name == "posix": + time.mktime(tuple(ltime)) # Does not crash anymore + else: + raises(OverflowError, time.mktime, tuple(ltime)) ltime[0] = 100 - time.mktime(tuple(ltime)) # Does not crash anymore + if os.name == "posix": + time.mktime(tuple(ltime)) # Does not crash anymore + else: + raises(OverflowError, time.mktime, tuple(ltime)) t = time.time() assert int(time.mktime(time.localtime(t))) == int(t) @@ -256,6 +262,8 @@ def test_localtime_timezone(self): import os, time + if not os.name == "posix": + skip("tzset available only under Unix") org_TZ = os.environ.get('TZ', None) try: os.environ['TZ'] = 'Europe/Kiev' @@ -414,6 +422,8 @@ def test_pep475_retry_sleep(self): import time import _signal as signal + if not hasattr(signal, 'SIGALRM'): + skip("SIGALRM available only under Unix") signalled = [] def foo(*args): From pypy.commits at gmail.com Fri Feb 23 04:33:31 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Feb 2018 01:33:31 -0800 (PST) Subject: [pypy-commit] cffi default: Import from setuptools instead of distutils from here Message-ID: <5a8fdfeb.91bddf0a.c78f1.b86b@mx.google.com> Author: Armin Rigo Branch: Changeset: r3105:1a92038e7904 Date: 2018-02-23 10:31 +0100 http://bitbucket.org/cffi/cffi/changeset/1a92038e7904/ Log: Import from setuptools instead of distutils from here (see issue #345) diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -148,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler From pypy.commits at gmail.com Fri Feb 23 08:41:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Feb 2018 05:41:40 -0800 (PST) Subject: [pypy-commit] pypy default: Document merged branch Message-ID: <5a901a14.4588df0a.b1d0c.defd@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93874:a17a3e4c087f Date: 2018-02-23 13:40 +0000 http://bitbucket.org/pypy/pypy/changeset/a17a3e4c087f/ Log: Document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -44,3 +44,7 @@ .. branch: msvc14 Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. From pypy.commits at gmail.com Fri Feb 23 09:21:15 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Feb 2018 06:21:15 -0800 (PST) Subject: [pypy-commit] pypy default: Use @signature to prevent late-stage annotation issues Message-ID: <5a90235b.cd151c0a.71f4d.92dd@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93875:b6c556e371ec Date: 2018-02-23 14:20 +0000 http://bitbucket.org/pypy/pypy/changeset/b6c556e371ec/ Log: Use @signature to prevent late-stage annotation issues diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -109,9 +109,9 @@ wchar_t const* file, unsigned int line, uintptr_t pReserved) { - wprintf(L"Invalid parameter detected in function %s." - L" File: %s Line: %d\\n", function, file, line); - wprintf(L"Expression: %s\\n", expression); + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); } RPY_EXTERN void* enter_suppress_iph(void) @@ -267,7 +267,7 @@ if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], rffi.INT, compilation_info=errno_eci, )) c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", @@ -515,7 +515,7 @@ releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('read') - at enforceargs(int, int) + at signature(types.int(), types.int(), returns=types.any()) def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) @@ -526,7 +526,7 @@ return buf.str(got) @replace_os_function('write') - at enforceargs(int, None) + at signature(types.int(), types.any(), returns=types.any()) def write(fd, data): count = len(data) with FdValidator(fd): @@ -536,6 +536,7 @@ return handle_posix_error('write', ret) @replace_os_function('close') + at signature(types.int(), returns=types.any()) def close(fd): with FdValidator(fd): handle_posix_error('close', c_close(fd)) From pypy.commits at gmail.com Fri Feb 23 09:45:57 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 23 Feb 2018 06:45:57 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: maybe work on rpython documentation at the sprint Message-ID: <5a902925.d6abdf0a.8564.1ac3@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5869:4190a252997f Date: 2018-02-23 16:45 -0500 http://bitbucket.org/pypy/extradoc/changeset/4190a252997f/ Log: maybe work on rpython documentation at the sprint diff --git a/planning/sprint-leysin-2018-notes.rst b/planning/sprint-leysin-2018-notes.rst --- a/planning/sprint-leysin-2018-notes.rst +++ b/planning/sprint-leysin-2018-notes.rst @@ -13,3 +13,4 @@ - finish unicode-utf8 - update www.pypy.org, speed.pypy.org (web devs needed) - go over projects at https://bitbucket.org/pypy, delete or document dead projects +- document the different rpython decorators like enforceargs, signature, and then interp-level unwrap_spec From pypy.commits at gmail.com Fri Feb 23 11:02:17 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Feb 2018 08:02:17 -0800 (PST) Subject: [pypy-commit] pypy default: Complain if we pass None to something which expects unicode or str Message-ID: <5a903b09.8bc7df0a.be369.868f@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93876:2ec2d28bba33 Date: 2018-02-23 16:01 +0000 http://bitbucket.org/pypy/pypy/changeset/2ec2d28bba33/ Log: Complain if we pass None to something which expects unicode or str diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -137,14 +137,11 @@ def decorator(f): def get_annotation(t): from rpython.annotator.signature import annotation - from rpython.annotator.model import SomeObject, SomeString, SomeUnicodeString + from rpython.annotator.model import SomeObject if isinstance(t, SomeObject): return t - s_result = annotation(t) - if (isinstance(s_result, SomeString) or - isinstance(s_result, SomeUnicodeString)): - return s_result.__class__(can_be_None=True) - return s_result + return annotation(t) + def get_type_descr_of_argument(arg): # we don't want to check *all* the items in list/dict: we assume # they are already homogeneous, so we only check the first diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -484,12 +484,6 @@ # in RPython there is an implicit int->float promotion assert f(42) == 42 -def test_enforceargs_None_string(): - @enforceargs(str, unicode) - def f(a, b): - return a, b - assert f(None, None) == (None, None) - def test_enforceargs_complex_types(): @enforceargs([int], {str: int}) def f(a, b): From pypy.commits at gmail.com Fri Feb 23 13:57:22 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Feb 2018 10:57:22 -0800 (PST) Subject: [pypy-commit] cffi default: Document issue #355 and #350 here Message-ID: <5a906412.45acdf0a.bf220.6f00@mx.google.com> Author: Armin Rigo Branch: Changeset: r3106:db50b63de204 Date: 2018-02-23 19:57 +0100 http://bitbucket.org/cffi/cffi/changeset/db50b63de204/ Log: Document issue #355 and #350 here diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -560,7 +560,13 @@ ``NAME.cpython-35m-x86_64-linux-gnu.so``. You can manually rename it to ``NAME.abi3.so``, or use setuptools version 26 or later. Also, note that compiling with a debug version of Python will not actually define -``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy. +``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy. Finally, +``Py_LIMITED_API`` is not defined on Windows, because this makes +modules which cannot be used with ``virtualenv`` (issues `#355`__ and +`#350`__). + +.. __: https://bitbucket.org/cffi/cffi/issues/355/importerror-dll-load-failed-on-windows +.. __: https://bitbucket.org/cffi/cffi/issues/350/issue-with-py_limited_api-on-windows **ffibuilder.compile(tmpdir='.', verbose=False, debug=None):** explicitly generate the .py or .c file, From pypy.commits at gmail.com Fri Feb 23 16:03:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Feb 2018 13:03:40 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Don't segfault when calling ffi.from_buffer() on io.BytesIO().getbuffer() Message-ID: <5a9081ac.4a8a1c0a.a5cae.76c4@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93877:baf4067a05b0 Date: 2018-02-23 21:02 +0000 http://bitbucket.org/pypy/pypy/changeset/baf4067a05b0/ Log: Don't segfault when calling ffi.from_buffer() on io.BytesIO().getbuffer() diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -288,6 +288,15 @@ ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + def test_from_buffer_BytesIO(self): + from _cffi_backend import FFI + import _io + ffi = FFI() + a = _io.BytesIO(b"Hello, world!") + buf = a.getbuffer() + # used to segfault + raises(TypeError, ffi.from_buffer, buf) + def test_memmove(self): import sys import _cffi_backend as _cffi1_backend diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -48,6 +48,9 @@ finally: w_bytesio.seek(tell) + def get_raw_address(self): + raise ValueError("BytesIOBuffer does not have a raw address") + class W_BytesIO(W_BufferedIOBase): import_from_mixin(RStringIO) From pypy.commits at gmail.com Sat Feb 24 03:26:58 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 24 Feb 2018 00:26:58 -0800 (PST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5a9121d2.87a4df0a.5a6df.7411@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93879:ebe1844aaf79 Date: 2018-02-24 09:26 +0100 http://bitbucket.org/pypy/pypy/changeset/ebe1844aaf79/ Log: merge heads diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -270,8 +270,8 @@ # try: if operr is not None: - ec = self.space.getexecutioncontext() - next_instr = self.handle_operation_error(ec, operr) + next_instr = self.handle_operation_error( + executioncontext, operr) self.last_instr = intmask(next_instr - 1) else: # Execution starts just after the last_instr. Initially, From pypy.commits at gmail.com Sat Feb 24 03:26:55 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 24 Feb 2018 00:26:55 -0800 (PST) Subject: [pypy-commit] pypy default: no need to get the executioncontext a second time, it's already in a local Message-ID: <5a9121cf.45a9df0a.43b7c.982f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93878:57349ce40416 Date: 2018-02-23 11:22 +0100 http://bitbucket.org/pypy/pypy/changeset/57349ce40416/ Log: no need to get the executioncontext a second time, it's already in a local variable diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -270,8 +270,8 @@ # try: if operr is not None: - ec = self.space.getexecutioncontext() - next_instr = self.handle_operation_error(ec, operr) + next_instr = self.handle_operation_error( + executioncontext, operr) self.last_instr = intmask(next_instr - 1) else: # Execution starts just after the last_instr. Initially, From pypy.commits at gmail.com Sat Feb 24 03:27:53 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 24 Feb 2018 00:27:53 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: a branch to try to emit less threadlocalref_get calls in bridges Message-ID: <5a912209.0ee51c0a.64f6c.e26d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93880:7d193b453f44 Date: 2018-02-23 21:31 +0100 http://bitbucket.org/pypy/pypy/changeset/7d193b453f44/ Log: a branch to try to emit less threadlocalref_get calls in bridges first failing test diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -321,3 +321,38 @@ self.check_trace_count(3) self.check_resops(guard_value=1) self.check_resops(getarrayitem_gc_i=5) + + def test_bridge_call_loopinvariant(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + a = get() + res += a.x + if y > n: + res += 1 + res += get().x + a.x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) From pypy.commits at gmail.com Sat Feb 24 03:27:55 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 24 Feb 2018 00:27:55 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: implement persisting call_loopinvariant knowledge across bridges Message-ID: <5a91220b.c12b1c0a.5240c.1eae@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93881:fbb6ecffb551 Date: 2018-02-24 09:25 +0100 http://bitbucket.org/pypy/pypy/changeset/fbb6ecffb551/ Log: implement persisting call_loopinvariant knowledge across bridges diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # # ( ) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # # ( ) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# +# ( ) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -27,6 +27,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +62,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +102,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) dct = {box: cls for box, known_class in boxes_known_classes @@ -347,6 +349,7 @@ while y > 0: myjitdriver.jit_merge_point(y=y, n=n, res=res) a = get() + a = get() res += a.x if y > n: res += 1 From pypy.commits at gmail.com Sat Feb 24 03:27:57 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 24 Feb 2018 00:27:57 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: merge default Message-ID: <5a91220d.9788df0a.3d351.db6f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93882:ebe462fa49b9 Date: 2018-02-24 09:27 +0100 http://bitbucket.org/pypy/pypy/changeset/ebe462fa49b9/ Log: merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -44,3 +44,7 @@ .. branch: msvc14 Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -137,14 +137,11 @@ def decorator(f): def get_annotation(t): from rpython.annotator.signature import annotation - from rpython.annotator.model import SomeObject, SomeString, SomeUnicodeString + from rpython.annotator.model import SomeObject if isinstance(t, SomeObject): return t - s_result = annotation(t) - if (isinstance(s_result, SomeString) or - isinstance(s_result, SomeUnicodeString)): - return s_result.__class__(can_be_None=True) - return s_result + return annotation(t) + def get_type_descr_of_argument(arg): # we don't want to check *all* the items in list/dict: we assume # they are already homogeneous, so we only check the first diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -109,9 +109,9 @@ wchar_t const* file, unsigned int line, uintptr_t pReserved) { - wprintf(L"Invalid parameter detected in function %s." - L" File: %s Line: %d\\n", function, file, line); - wprintf(L"Expression: %s\\n", expression); + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); } RPY_EXTERN void* enter_suppress_iph(void) @@ -267,7 +267,7 @@ if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], rffi.INT, compilation_info=errno_eci, )) c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", @@ -515,7 +515,7 @@ releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('read') - at enforceargs(int, int) + at signature(types.int(), types.int(), returns=types.any()) def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) @@ -526,7 +526,7 @@ return buf.str(got) @replace_os_function('write') - at enforceargs(int, None) + at signature(types.int(), types.any(), returns=types.any()) def write(fd, data): count = len(data) with FdValidator(fd): @@ -536,6 +536,7 @@ return handle_posix_error('write', ret) @replace_os_function('close') + at signature(types.int(), returns=types.any()) def close(fd): with FdValidator(fd): handle_posix_error('close', c_close(fd)) diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -484,12 +484,6 @@ # in RPython there is an implicit int->float promotion assert f(42) == 42 -def test_enforceargs_None_string(): - @enforceargs(str, unicode) - def f(a, b): - return a, b - assert f(None, None) == (None, None) - def test_enforceargs_complex_types(): @enforceargs([int], {str: int}) def f(a, b): From pypy.commits at gmail.com Sat Feb 24 12:00:12 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 24 Feb 2018 09:00:12 -0800 (PST) Subject: [pypy-commit] pypy default: Issue #2758 Message-ID: <5a919a1c.480f1c0a.2eedf.359d@mx.google.com> Author: Armin Rigo Branch: Changeset: r93883:89245c285fa0 Date: 2018-02-24 17:59 +0100 http://bitbucket.org/pypy/pypy/changeset/89245c285fa0/ Log: Issue #2758 Test and fix for re.sub(callback) receiving an immutable match object diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -303,8 +303,11 @@ if not (last_pos == ctx.match_start == ctx.match_end and n > 0): # the above ignores empty matches on latest position + last_pos = ctx.match_end if filter_is_callable: w_match = self.getmatch(ctx, True) + # make a copy of 'ctx'; see test_sub_matches_stay_valid + ctx = ctx.fresh_copy(start) # match_start/match_end dropped w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None and unicodebuilder is None @@ -321,7 +324,6 @@ unicodebuilder.append(filter_as_unicode) else: sublist_w.append(w_filter) - last_pos = ctx.match_end n += 1 elif last_pos >= ctx.end: break # empty match at the end: finished diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -328,6 +328,18 @@ KEYCRE = re.compile(r"%\(([^)]*)\)s|.") raises(TypeError, KEYCRE.sub, "hello", {"%(": 1}) + def test_sub_matches_stay_valid(self): + import re + matches = [] + def callback(match): + matches.append(match) + return "x" + result = re.compile(r"[ab]").sub(callback, "acb") + assert result == "xcx" + assert len(matches) == 2 + assert matches[0].group() == "a" + assert matches[1].group() == "b" + class AppTestSreScanner: def test_scanner_attributes(self): From pypy.commits at gmail.com Sun Feb 25 13:16:30 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Feb 2018 10:16:30 -0800 (PST) Subject: [pypy-commit] pypy py3.5: allow directly using VsDevCmd.bat for newer Visual Studio Build Tools installations Message-ID: <5a92fd7e.dd8bdf0a.2faf5.37ed@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93884:7e7871873b7f Date: 2018-02-25 20:03 -0500 http://bitbucket.org/pypy/pypy/changeset/7e7871873b7f/ Log: allow directly using VsDevCmd.bat for newer Visual Studio Build Tools installations diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -233,11 +233,17 @@ if not productdir or not os.path.isdir(productdir): toolskey = "VS%0.f0COMNTOOLS" % version toolsdir = os.environ.get(toolskey, None) + batfile = 'vcvarsall.bat' if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) + if os.path.exists(os.path.join(toolsdir, 'VsDevCmd.bat')): + productdir = toolsdir + batfile = 'VsDevCmd.bat' + else: + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) return None else: @@ -245,7 +251,7 @@ if not productdir: log.debug("No productdir found") return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") + vcvarsall = os.path.join(productdir, batfile) if os.path.isfile(vcvarsall): return vcvarsall log.debug("Unable to find vcvarsall.bat") From pypy.commits at gmail.com Sun Feb 25 13:16:32 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Feb 2018 10:16:32 -0800 (PST) Subject: [pypy-commit] pypy default: document 7e7871873b7f for finding visual studio compilers Message-ID: <5a92fd80.01c0df0a.18aca.4d15@mx.google.com> Author: Matti Picus Branch: Changeset: r93885:9cd3ff760593 Date: 2018-02-25 20:15 -0500 http://bitbucket.org/pypy/pypy/changeset/9cd3ff760593/ Log: document 7e7871873b7f for finding visual studio compilers diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -554,6 +554,11 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: `distutils` has been enhanced to allow finding `VsDevCmd.bat` in the + directory pointed to by the `VS%0.f0COMNTOOLS` (typically `VS140COMNTOOLS`) + environment variable. CPython searches for `vcvarsall.bat` somewhere _above_ + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ From pypy.commits at gmail.com Sun Feb 25 13:25:20 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Feb 2018 10:25:20 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5a92ff90.c4d51c0a.f406d.3029@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93886:6dcfe4a00bb7 Date: 2018-02-25 20:24 -0500 http://bitbucket.org/pypy/pypy/changeset/6dcfe4a00bb7/ Log: merge default into branch diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py --- a/lib_pypy/_dbm.py +++ b/lib_pypy/_dbm.py @@ -168,7 +168,14 @@ def open(filename, flag='r', mode=0o666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") filename = filename.encode(sys.getdefaultencoding()) openflag = 0 diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -567,6 +567,11 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: `distutils` has been enhanced to allow finding `VsDevCmd.bat` in the + directory pointed to by the `VS%0.f0COMNTOOLS` (typically `VS140COMNTOOLS`) + environment variable. CPython searches for `vcvarsall.bat` somewhere _above_ + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,7 @@ .. branch: winapi Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -385,8 +385,11 @@ if not (last_pos == ctx.match_start == ctx.match_end and n > 0): # the above ignores empty matches on latest position + last_pos = ctx.match_end if filter_is_callable: w_match = self.getmatch(ctx, True) + # make a copy of 'ctx'; see test_sub_matches_stay_valid + ctx = ctx.fresh_copy(start) # match_start/match_end dropped w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None and unicodebuilder is None @@ -403,7 +406,6 @@ unicodebuilder.append(filter_as_unicode) else: sublist_w.append(w_filter) - last_pos = ctx.match_end n += 1 elif last_pos >= ctx.end: break # empty match at the end: finished diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -355,6 +355,18 @@ KEYCRE = re.compile(r"%\(([^)]*)\)s|.") raises(TypeError, KEYCRE.sub, "hello", {"%(": 1}) + def test_sub_matches_stay_valid(self): + import re + matches = [] + def callback(match): + matches.append(match) + return "x" + result = re.compile(r"[ab]").sub(callback, "acb") + assert result == "xcx" + assert len(matches) == 2 + assert matches[0].group() == "a" + assert matches[1].group() == "b" + class AppTestSreScanner: diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/pypy/module/test_lib_pypy/test_dbm_extra.py @@ -1,4 +1,4 @@ -import py +import py, os from rpython.tool.udir import udir try: from lib_pypy import dbm @@ -73,3 +73,8 @@ assert 'key_with_empty_value' in d assert d['key_with_empty_value'] == '' d.close() + +def test_unicode_filename(): + path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' + d = dbm.open(path, 'c') + d.close() diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -354,7 +354,7 @@ else: # this is dead code, but in case we have a gc that does # not have a write barrier and does not zero memory, we would - # need to clal it + # need to call it if op.getopnum() == rop.SETFIELD_GC: self.consider_setfield_gc(op) elif op.getopnum() == rop.SETARRAYITEM_GC: diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -30,7 +30,7 @@ class CompileData(object): memo = None log_noopt = True - + def forget_optimization_info(self): for arg in self.trace.inputargs: arg.set_forwarded(None) @@ -67,19 +67,26 @@ """ This represents label() ops jump with no extra info associated with the label """ - def __init__(self, trace, call_pure_results=None, + def __init__(self, trace, resumestorage=None, call_pure_results=None, enable_opts=None): self.trace = trace + self.resumestorage = resumestorage self.call_pure_results = call_pure_results self.enable_opts = enable_opts def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer + from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace.get_iter(), - self.call_pure_results) + traceiter = self.trace.get_iter() + if self.resumestorage: + frontend_inputargs = self.trace.inputargs + deserialize_optimizer_knowledge(opt, self.resumestorage, + frontend_inputargs, + traceiter.inputargs) + return opt.propagate_all_forward(traceiter, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some @@ -518,7 +525,7 @@ for item in lst: item.set_forwarded(None) # XXX we should really do it, but we need to remember the values - # somehoe for ContinueRunningNormally + # somehow for ContinueRunningNormally if reset_values: item.reset_value() @@ -671,38 +678,16 @@ raise jitexc.ExitFrameWithExceptionRef(cpu, value) -class TerminatingLoopToken(JitCellToken): # FIXME: kill? - terminating = True - - def __init__(self, nargs, finishdescr): - self.finishdescr = finishdescr - -def make_done_loop_tokens(): - done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() - done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() - done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() - done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() - exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() - - # pseudo loop tokens to make the life of optimize.py easier - d = {'loop_tokens_done_with_this_frame_int': [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ], - 'loop_tokens_done_with_this_frame_ref': [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ], - 'loop_tokens_done_with_this_frame_float': [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ], - 'loop_tokens_done_with_this_frame_void': [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ], - 'loop_tokens_exit_frame_with_exception_ref': [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ], - } - d.update(locals()) - return d +def make_and_attach_done_descrs(targets): + for name, cls in [ + ("done_with_this_frame_descr_void", DoneWithThisFrameDescrVoid), + ("done_with_this_frame_descr_int", DoneWithThisFrameDescrInt), + ("done_with_this_frame_descr_ref", DoneWithThisFrameDescrRef), + ("done_with_this_frame_descr_float", DoneWithThisFrameDescrFloat), + ("exit_frame_with_exception_descr_ref", ExitFrameWithExceptionDescrRef)]: + descr = cls() + for target in targets: + setattr(target, name, descr) class ResumeDescr(AbstractFailDescr): _attrs_ = () @@ -726,6 +711,9 @@ TY_REF = 0x04 TY_FLOAT = 0x06 + def get_resumestorage(self): + raise NotImplementedError("abstract base class") + def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd) and not rstack.stack_almost_full()): @@ -854,15 +842,23 @@ class ResumeGuardCopiedDescr(AbstractResumeGuardDescr): _attrs_ = ('status', 'prev') + def __init__(self, prev): + AbstractResumeGuardDescr.__init__(self) + assert isinstance(prev, ResumeGuardDescr) + self.prev = prev + def copy_all_attributes_from(self, other): assert isinstance(other, ResumeGuardCopiedDescr) self.prev = other.prev def clone(self): - cloned = ResumeGuardCopiedDescr() - cloned.copy_all_attributes_from(self) + cloned = ResumeGuardCopiedDescr(self.prev) return cloned + def get_resumestorage(self): + prev = self.prev + assert isinstance(prev, ResumeGuardDescr) + return prev class ResumeGuardDescr(AbstractResumeGuardDescr): _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', @@ -873,8 +869,7 @@ rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) def copy_all_attributes_from(self, other): - if isinstance(other, ResumeGuardCopiedDescr): - other = other.prev + other = other.get_resumestorage() assert isinstance(other, ResumeGuardDescr) self.rd_consts = other.rd_consts self.rd_pendingfields = other.rd_pendingfields @@ -895,6 +890,9 @@ cloned.copy_all_attributes_from(self) return cloned + def get_resumestorage(self): + return self + class ResumeGuardExcDescr(ResumeGuardDescr): pass @@ -936,22 +934,22 @@ ptr = cpu.ts.cast_to_baseclass(gcref) return cast_base_ptr_to_instance(AllVirtuals, ptr) -def invent_fail_descr_for_op(opnum, optimizer, copied_guard=False): +def invent_fail_descr_for_op(opnum, optimizer, copied_from_descr=None): if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - assert not copied_guard + assert copied_from_descr is None resumedescr = ResumeGuardForcedDescr() resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd) elif opnum in (rop.GUARD_IS_OBJECT, rop.GUARD_SUBCLASS, rop.GUARD_GC_TYPE): # note - this only happens in tests resumedescr = ResumeAtPositionDescr() elif opnum in (rop.GUARD_EXCEPTION, rop.GUARD_NO_EXCEPTION): - if copied_guard: - resumedescr = ResumeGuardCopiedExcDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedExcDescr(copied_from_descr) else: resumedescr = ResumeGuardExcDescr() else: - if copied_guard: - resumedescr = ResumeGuardCopiedDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedDescr(copied_from_descr) else: resumedescr = ResumeGuardDescr() return resumedescr @@ -1036,6 +1034,9 @@ self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + def get_resumestorage(self): + return None + def compile_trace(metainterp, resumekey, runtime_boxes): """Try to compile a new bridge leading from the beginning of the history @@ -1067,22 +1068,15 @@ enable_opts = jitdriver_sd.warmstate.enable_opts call_pure_results = metainterp.call_pure_results + resumestorage = resumekey.get_resumestorage() if metainterp.history.ends_with_jump: - if isinstance(resumekey, ResumeGuardCopiedDescr): - key = resumekey.prev - assert isinstance(key, ResumeGuardDescr) - elif isinstance(resumekey, ResumeFromInterpDescr): - key = None - else: - key = resumekey - assert isinstance(key, ResumeGuardDescr) - data = BridgeCompileData(trace, runtime_boxes, key, + data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, inline_short_preamble=inline_short_preamble) else: - data = SimpleCompileData(trace, + data = SimpleCompileData(trace, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts) try: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -404,7 +404,6 @@ target_tokens = None failed_states = None retraced_count = 0 - terminating = False # see TerminatingLoopToken in compile.py invalidated = False outermost_jitdriver_sd = None # and more data specified by the backend when the loop is compiled @@ -935,7 +934,7 @@ return insns def check_simple_loop(self, expected=None, **check): - """ Usefull in the simplest case when we have only one trace ending with + """ Useful in the simplest case when we have only one trace ending with a jump back to itself and possibly a few bridges. Only the operations within the loop formed by that single jump will be counted. diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -688,12 +688,10 @@ def _copy_resume_data_from(self, guard_op, last_guard_op): - descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, True) last_descr = last_guard_op.getdescr() + descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) assert isinstance(last_descr, compile.ResumeGuardDescr) - if isinstance(descr, compile.ResumeGuardCopiedDescr): - descr.prev = last_descr - else: + if not isinstance(descr, compile.ResumeGuardCopiedDescr): descr.copy_all_attributes_from(last_descr) guard_op.setdescr(descr) guard_op.setfailargs(last_guard_op.getfailargs()) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -31,8 +31,8 @@ expected = convert_old_style_to_targets(exp, jump=True) call_pure_results = self._convert_call_pure_results(call_pure_results) trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu)) - compile_data = compile.SimpleCompileData(trace, - call_pure_results) + compile_data = compile.SimpleCompileData( + trace, call_pure_results=call_pure_results) info, ops = self._do_optimize_loop(compile_data) label_op = ResOperation(rop.LABEL, info.inputargs) loop.inputargs = info.inputargs diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1854,12 +1854,7 @@ self._addr2name_keys = [] self._addr2name_values = [] - self.__dict__.update(compile.make_done_loop_tokens()) - for val in ['int', 'float', 'ref', 'void']: - fullname = 'done_with_this_frame_descr_' + val - setattr(self.cpu, fullname, getattr(self, fullname)) - d = self.exit_frame_with_exception_descr_ref - self.cpu.exit_frame_with_exception_descr_ref = d + compile.make_and_attach_done_descrs([self, cpu]) def _freeze_(self): return True @@ -1909,8 +1904,8 @@ history.REF: 'ref', history.FLOAT: 'float', history.VOID: 'void'}[jd.result_type] - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - jd.portal_finishtoken = tokens[0].finishdescr + token = getattr(self, 'done_with_this_frame_descr_%s' % name) + jd.portal_finishtoken = token jd.propagate_exc_descr = exc_descr # self.cpu.propagate_exception_descr = exc_descr @@ -2463,10 +2458,7 @@ def handle_guard_failure(self, resumedescr, deadframe): debug_start('jit-tracing') self.staticdata.profiler.start_tracing() - if isinstance(resumedescr, compile.ResumeGuardCopiedDescr): - key = resumedescr.prev - else: - key = resumedescr + key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp @@ -2770,21 +2762,19 @@ if result_type == history.VOID: assert exitbox is None exits = [] - loop_tokens = sd.loop_tokens_done_with_this_frame_void + token = sd.done_with_this_frame_descr_void elif result_type == history.INT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_int + token = sd.done_with_this_frame_descr_int elif result_type == history.REF: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_ref + token = sd.done_with_this_frame_descr_ref elif result_type == history.FLOAT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_float + token = sd.done_with_this_frame_descr_float else: assert False - # FIXME: kill TerminatingLoopToken? # FIXME: can we call compile_trace? - token = loop_tokens[0].finishdescr self.history.record(rop.FINISH, exits, None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, exits) @@ -2810,7 +2800,7 @@ def compile_exit_frame_with_exception(self, valuebox): self.store_token_in_vable() sd = self.staticdata - token = sd.loop_tokens_exit_frame_with_exception_ref[0].finishdescr + token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -143,6 +143,42 @@ self.check_trace_count(3) self.check_resops(guard_class=1) + def test_bridge_guard_class_return(self): + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) + class A(object): + def f(self): + return 1 + class B(A): + def f(self): + return 2 + def f(x, y, n): + if x: + a = A() + else: + a = B() + a.x = 0 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) + res += a.f() + a.x += 1 + if y < n: + res += 1 + res += a.f() + return res + res += a.f() + y -= 1 + return res + def g(i): + res = 0 + for i in range(i): + res += f(6, 32, 16-i) + res1 = g(10) + res2 = self.meta_interp(g, [10]) + assert res1 == res2 + self.check_trace_count(2) + self.check_resops(guard_class=1, omit_finish=False) + def test_bridge_field_read(self): myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) class A(object): diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -137,14 +137,11 @@ def decorator(f): def get_annotation(t): from rpython.annotator.signature import annotation - from rpython.annotator.model import SomeObject, SomeString, SomeUnicodeString + from rpython.annotator.model import SomeObject if isinstance(t, SomeObject): return t - s_result = annotation(t) - if (isinstance(s_result, SomeString) or - isinstance(s_result, SomeUnicodeString)): - return s_result.__class__(can_be_None=True) - return s_result + return annotation(t) + def get_type_descr_of_argument(arg): # we don't want to check *all* the items in list/dict: we assume # they are already homogeneous, so we only check the first diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -109,9 +109,9 @@ wchar_t const* file, unsigned int line, uintptr_t pReserved) { - wprintf(L"Invalid parameter detected in function %s." - L" File: %s Line: %d\\n", function, file, line); - wprintf(L"Expression: %s\\n", expression); + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); } RPY_EXTERN void* enter_suppress_iph(void) @@ -267,7 +267,7 @@ if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], rffi.INT, compilation_info=errno_eci, )) c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", @@ -515,7 +515,7 @@ releasegil=False, save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('read') - at enforceargs(int, int) + at signature(types.int(), types.int(), returns=types.any()) def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) @@ -526,7 +526,7 @@ return buf.str(got) @replace_os_function('write') - at enforceargs(int, None) + at signature(types.int(), types.any(), returns=types.any()) def write(fd, data): count = len(data) with FdValidator(fd): @@ -536,6 +536,7 @@ return handle_posix_error('write', ret) @replace_os_function('close') + at signature(types.int(), returns=types.any()) def close(fd): with FdValidator(fd): handle_posix_error('close', c_close(fd)) diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -484,12 +484,6 @@ # in RPython there is an implicit int->float promotion assert f(42) == 42 -def test_enforceargs_None_string(): - @enforceargs(str, unicode) - def f(a, b): - return a, b - assert f(None, None) == (None, None) - def test_enforceargs_complex_types(): @enforceargs([int], {str: int}) def f(a, b): From pypy.commits at gmail.com Sun Feb 25 13:57:28 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 25 Feb 2018 10:57:28 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add _blake2 to the build_cffi_imports script. Message-ID: <5a930718.8bc7df0a.be369.bc50@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93887:6f93003631b8 Date: 2018-02-25 19:52 +0100 http://bitbucket.org/pypy/pypy/changeset/6f93003631b8/ Log: Add _blake2 to the build_cffi_imports script. diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -41,8 +41,8 @@ """ blake2b_source = """ -#include "blake2.h" -#include "blake2-impl.h" +#include "impl/blake2.h" +#include "impl/blake2-impl.h" #define BLAKE_OUTBYTES BLAKE2B_OUTBYTES #define BLAKE_SALTBYTES BLAKE2B_SALTBYTES @@ -63,7 +63,7 @@ """ -_libdir = os.path.join(os.path.dirname(__file__), 'impl') +_libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'impl')) blake2b_ffi = FFI() blake2b_ffi.cdef(blake_cdef) diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -18,7 +18,8 @@ "lzma": "_lzma_build.py", "_decimal": "_decimal_build.py", "_ssl": "_ssl_build.py", - # hashlib does not need to be built! It uses API calls from _ssl + "_blake2": "_blake2/_blake2_build.py", + "_sha3": "_sha3/_sha3_build.py", "xx": None, # for testing: 'None' should be completely ignored } From pypy.commits at gmail.com Sun Feb 25 14:48:01 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Feb 2018 11:48:01 -0800 (PST) Subject: [pypy-commit] pypy default: formatting Message-ID: <5a9312f1.12691c0a.682c6.8893@mx.google.com> Author: Matti Picus Branch: Changeset: r93888:15dc09ae89b8 Date: 2018-02-25 21:47 -0500 http://bitbucket.org/pypy/pypy/changeset/15dc09ae89b8/ Log: formatting diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -554,9 +554,9 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. -* PyPy3: `distutils` has been enhanced to allow finding `VsDevCmd.bat` in the - directory pointed to by the `VS%0.f0COMNTOOLS` (typically `VS140COMNTOOLS`) - environment variable. CPython searches for `vcvarsall.bat` somewhere _above_ +* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in the + directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically ``VS140COMNTOOLS``) + environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 From pypy.commits at gmail.com Sun Feb 25 16:16:58 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 25 Feb 2018 13:16:58 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Trigger gc collection after "del", to unblock the test. Message-ID: <5a9327ca.02ba1c0a.2d68b.0255@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93889:6379ab551ae1 Date: 2018-02-25 22:01 +0100 http://bitbucket.org/pypy/pypy/changeset/6379ab551ae1/ Log: Trigger gc collection after "del", to unblock the test. diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py --- a/lib-python/3/test/test_concurrent_futures.py +++ b/lib-python/3/test/test_concurrent_futures.py @@ -161,6 +161,7 @@ executor.map(abs, range(-5, 5)) threads = executor._threads del executor + test.support.gc_collect() for t in threads: self.assertRegex(t.name, r'^SpecialPool_[0-4]$') @@ -171,6 +172,7 @@ executor.map(abs, range(-5, 5)) threads = executor._threads del executor + test.support.gc_collect() for t in threads: # We don't particularly care what the default name is, just that From pypy.commits at gmail.com Sun Feb 25 16:17:00 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 25 Feb 2018 13:17:00 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Restore necessary import Message-ID: <5a9327cc.8f98df0a.c8565.e1d6@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93890:cf1579e0d8fa Date: 2018-02-25 22:16 +0100 http://bitbucket.org/pypy/pypy/changeset/cf1579e0d8fa/ Log: Restore necessary import diff --git a/lib-python/3/ctypes/test/test_parameters.py b/lib-python/3/ctypes/test/test_parameters.py --- a/lib-python/3/ctypes/test/test_parameters.py +++ b/lib-python/3/ctypes/test/test_parameters.py @@ -1,4 +1,4 @@ -import unittest +import unittest, sys from ctypes.test import need_symbol from ctypes.test import xfail From pypy.commits at gmail.com Mon Feb 26 12:08:56 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Feb 2018 09:08:56 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: fix issue #2765 Message-ID: <5a943f28.ef9e500a.68c30.0d50@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r915:fa51d386eb32 Date: 2018-02-26 19:08 -0500 http://bitbucket.org/pypy/pypy.org/changeset/fa51d386eb32/ Log: fix issue #2765 diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -84,8 +84,6 @@ (32/64 bits), Mac OS X (64 bits), Windows (32 bits), OpenBSD, FreeBSD. All non-x86 versions are only supported on Linux.

If you are interested in helping to move forward, see our howtohelp page.

-

We also have a beta release of PyPy3 which implements Python 3.2.5. -It runs on the same platforms as above.

The main features of PyPy:

diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -30,9 +30,6 @@ If you are interested in helping to move forward, see our `howtohelp`_ page. -We also have a beta release of **PyPy3** which implements **Python 3.2.5**. -It runs on the same platforms as above. - .. _`compatibility`: compat.html .. _`x86 (IA-32)`: http://en.wikipedia.org/wiki/IA-32 .. _`x86_64`: http://en.wikipedia.org/wiki/X86_64 From pypy.commits at gmail.com Mon Feb 26 15:41:25 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Feb 2018 12:41:25 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix a test Message-ID: <5a9470f5.510c1c0a.72515.f952@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93891:ac0bb5a87469 Date: 2018-02-26 21:24 -0500 http://bitbucket.org/pypy/pypy/changeset/ac0bb5a87469/ Log: fix a test diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -113,6 +113,7 @@ def test_pytrace(): output = run(sys.executable, pypypath, '-S', stdin="__pytrace__ = 1\nx = 5\nx") + output = output.replace('\r\n', '\n') assert ('\t: LOAD_CONST 0 (5)\n' '\t: STORE_NAME 0 (x)\n' '\t: LOAD_CONST 1 (None)\n' From pypy.commits at gmail.com Mon Feb 26 15:41:28 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Feb 2018 12:41:28 -0800 (PST) Subject: [pypy-commit] pypy py3.5: support finding vc14.1, vc15.0 Message-ID: <5a9470f8.51951c0a.9e45.bb23@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93892:751419a5d521 Date: 2018-02-26 21:24 -0500 http://bitbucket.org/pypy/pypy/changeset/751419a5d521/ Log: support finding vc14.1, vc15.0 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -46,7 +46,7 @@ arch = 'x64' else: arch = 'x86' - if version == 140: + if version >= 140: return msvc.msvc14_get_vc_env(arch) else: return msvc.msvc9_query_vcvarsall(version / 10.0, arch) @@ -115,7 +115,7 @@ return env def find_msvc_env(x64flag=False, ver0=None): - vcvers = [140, 90, 100] + vcvers = [140, 141, 150, 90, 100] if ver0 in vcvers: vcvers.insert(0, ver0) errs = [] From pypy.commits at gmail.com Mon Feb 26 15:41:30 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Feb 2018 12:41:30 -0800 (PST) Subject: [pypy-commit] pypy py3.5: skip _posixsubprocess tests on win32 Message-ID: <5a9470fa.3ab8500a.6a9a.84c0@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93893:ec983c59504a Date: 2018-02-26 21:56 -0500 http://bitbucket.org/pypy/pypy/changeset/ec983c59504a/ Log: skip _posixsubprocess tests on win32 diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py --- a/pypy/module/_posixsubprocess/test/test_subprocess.py +++ b/pypy/module/_posixsubprocess/test/test_subprocess.py @@ -1,4 +1,9 @@ from os.path import dirname +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") + class AppTestSubprocess: spaceconfig = dict(usemodules=('_posixsubprocess', 'signal', diff --git a/pypy/module/_posixsubprocess/test/test_ztranslation.py b/pypy/module/_posixsubprocess/test/test_ztranslation.py --- a/pypy/module/_posixsubprocess/test/test_ztranslation.py +++ b/pypy/module/_posixsubprocess/test/test_ztranslation.py @@ -1,4 +1,8 @@ from pypy.objspace.fake.checkmodule import checkmodule +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") def test_posixsubprocess_translates(): checkmodule('_posixsubprocess') From pypy.commits at gmail.com Mon Feb 26 16:22:53 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Feb 2018 13:22:53 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: next test I'd like to pass somehow Message-ID: <5a947aad.45a9df0a.43b7c.4cba@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93895:4a4d26fec6fc Date: 2018-02-26 20:16 +0100 http://bitbucket.org/pypy/pypy/changeset/4a4d26fec6fc/ Log: next test I'd like to pass somehow diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -359,3 +359,37 @@ res = self.meta_interp(f, [6, 32, 16]) self.check_trace_count(3) self.check_resops(call_r=1) + + def test_bridge_call_loopinvariant_2(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + if y > n: + res += get().x + res += 1 + res += get().x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) From pypy.commits at gmail.com Mon Feb 26 16:22:55 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Feb 2018 13:22:55 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: lift the restriction that cached heap fields that are stored across bridges Message-ID: <5a947aaf.c5b71c0a.c8cff.11d5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93896:89f271b2dd38 Date: 2018-02-26 22:22 +0100 http://bitbucket.org/pypy/pypy/changeset/89f271b2dd38/ Log: lift the restriction that cached heap fields that are stored across bridges have to be in liveboxes_from_env diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -33,13 +33,6 @@ # maybe should be delegated to the optimization classes? -def tag_box(box, liveboxes_from_env, memo): - from rpython.jit.metainterp.history import Const - if isinstance(box, Const): - return memo.getconst(box) - else: - return liveboxes_from_env[box] # has to exist - def decode_box(resumestorage, tagged, liveboxes, cpu): from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq @@ -60,11 +53,12 @@ raise AssertionError("unreachable") return box -def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): +def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, adder): from rpython.jit.metainterp.history import ConstInt + available_boxes = {} for box in liveboxes: - if box is not None and box in liveboxes_from_env: + if box is not None: available_boxes[box] = None # class knowledge is stored as bits, true meaning the class is known, false @@ -98,16 +92,16 @@ numb_state.append_int(len(triples_struct)) for box1, descr, box2 in triples_struct: descr_index = descr.descr_index - numb_state.append_short(tag_box(box1, liveboxes_from_env, memo)) + numb_state.append_short(adder._gettagged(box1)) numb_state.append_int(descr_index) - numb_state.append_short(tag_box(box2, liveboxes_from_env, memo)) + numb_state.append_short(adder._gettagged(box2)) numb_state.append_int(len(triples_array)) for box1, index, descr, box2 in triples_array: descr_index = descr.descr_index - numb_state.append_short(tag_box(box1, liveboxes_from_env, memo)) + numb_state.append_short(adder._gettagged(box1)) numb_state.append_int(index) numb_state.append_int(descr_index) - numb_state.append_short(tag_box(box2, liveboxes_from_env, memo)) + numb_state.append_short(adder._gettagged(box2)) else: numb_state.append_int(0) numb_state.append_int(0) @@ -118,8 +112,8 @@ numb_state.append_int(len(tuples_loopinvariant)) for constarg0, box in tuples_loopinvariant: numb_state.append_short( - tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) - numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + adder._gettagged(ConstInt(constarg0))) + numb_state.append_short(adder._gettagged(box)) else: numb_state.append_int(0) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -464,7 +464,7 @@ numb_state.patch(1, len(liveboxes)) - self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env) + self._add_optimizer_sections(numb_state, liveboxes) storage.rd_numb = numb_state.create_numbering() storage.rd_consts = self.memo.consts return liveboxes[:] @@ -584,11 +584,11 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] - def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env): + def _add_optimizer_sections(self, numb_state, liveboxes): # add extra information about things the optimizer learned from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge serialize_optimizer_knowledge( - self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo) + self.optimizer, numb_state, liveboxes, self) class AbstractVirtualInfo(object): kind = REF diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -60,7 +60,7 @@ numb_state.append_int(1) # size of resume block liveboxes = [InputArgInt(), box2, box1, box3] - serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, None) assert unpack_numbering(numb_state.create_numbering()) == [ 1, 0b010000, 0, 0, 0] @@ -100,7 +100,7 @@ numb_state.append_int(1) # size of resume block liveboxes = [box for (box, _) in boxes_known_classes] - serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, None) assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) @@ -221,6 +221,31 @@ self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n self.check_resops(getfield_gc_r=1) # in main loop + def test_bridge_field_read_virtual(self): + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) + class A(object): + pass + class Virt(object): + def __init__(self, n1): + self.n1 = n1 + + def f(y, n): + a = A() + a.n = n + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) + v = Virt(a.n) + if y > n: + res += 1 + res += v.n1 + a.n + y -= 1 + return res + res = self.meta_interp(f, [32, 16]) + assert res == f(32, 16) + self.check_trace_count(3) + self.check_resops(getfield_gc_i=1) # 1x a.x + def test_bridge_field_read_constants(self): myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) class A(object): From pypy.commits at gmail.com Mon Feb 26 16:22:50 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Feb 2018 13:22:50 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: fix test Message-ID: <5a947aaa.c9c4df0a.6f884.f9c7@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93894:8fd1d7d0d6a1 Date: 2018-02-26 20:14 +0100 http://bitbucket.org/pypy/pypy/changeset/8fd1d7d0d6a1/ Log: fix test diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -40,7 +40,7 @@ class FakeOptimizer(object): metainterp_sd = None - optheap = None + optheap = optrewrite = None def __init__(self, trace=None): self.trace = trace From pypy.commits at gmail.com Tue Feb 27 11:43:32 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 08:43:32 -0800 (PST) Subject: [pypy-commit] cffi default: Update whatsnew Message-ID: <5a958ab4.4dd91c0a.1a94.e060@mx.google.com> Author: Armin Rigo Branch: Changeset: r3107:fc38463f8f66 Date: 2018-02-27 17:43 +0100 http://bitbucket.org/cffi/cffi/changeset/fc38463f8f66/ Log: Update whatsnew diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -12,20 +12,28 @@ * Windows: ``ffi.dlopen()`` should now handle unicode filenames. -* Implemented ``ffi.dlclose()`` for the in-line case (it used to be - present only in the out-of-line case). +* ABI mode: implemented ``ffi.dlclose()`` for the in-line case (it used + to be present only in the out-of-line case). * Fixed a corner case for ``setup.py install --record=xx --root=yy`` - with an out-of-line ABI module. + with an out-of-line ABI module. Also fixed `Issue #345`_. * More hacks on Windows for running CFFI's own ``setup.py``. -* `Issue #358`_: in embedding, to protect against Python initialization - from several threads in parallel, we have to use a spin-lock. On - CPython 3 it is worse because it might spin-lock for a long time - (execution of ``Py_InitializeEx()``). Sadly, recent changes to - CPython make that solution needed on CPython 2 too. +* `Issue #358`_: in embedding, to protect against (the rare case of) + Python initialization from several threads in parallel, we have to use + a spin-lock. On CPython 3 it is worse because it might spin-lock for + a long time (execution of ``Py_InitializeEx()``). Sadly, recent + changes to CPython make that solution needed on CPython 2 too. +* CPython 3 on Windows: we no longer compile with ``Py_LIMITED_API`` + by default because such modules cannot be used with virtualenv. + `Issue #350`_ mentions a workaround if you still want that and are not + concerned about virtualenv: pass a ``define_macros=[("Py_LIMITED_API", + None)]`` to the ``ffibuilder.set_source()`` call. + +.. _`Issue #345`: https://bitbucket.org/cffi/cffi/issues/345/ +.. _`Issue #350`: https://bitbucket.org/cffi/cffi/issues/350/ .. _`Issue #358`: https://bitbucket.org/cffi/cffi/issues/358/ .. _`Issue #357`: https://bitbucket.org/cffi/cffi/issues/357/ From pypy.commits at gmail.com Tue Feb 27 13:15:23 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:15:23 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: hg merge default Message-ID: <5a95a03b.0aaedf0a.d8abd.a483@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3108:5ff833c65f91 Date: 2018-02-27 19:12 +0100 http://bitbucket.org/cffi/cffi/changeset/5ff833c65f91/ Log: hg merge default diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3794,27 +3794,29 @@ static int check_bytes_for_float_compatible(PyObject *io, double *out_value) { if (PyBytes_Check(io)) { - if (PyBytes_GET_SIZE(io) != 1) { - Py_DECREF(io); - return -1; - } + if (PyBytes_GET_SIZE(io) != 1) + goto error; *out_value = (unsigned char)PyBytes_AS_STRING(io)[0]; return 1; } else if (PyUnicode_Check(io)) { char ignored[80]; cffi_char32_t ordinal; - if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) { - Py_DECREF(io); - return -1; - } + if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) + goto error; /* the signness of the 32-bit version of wide chars should not * matter here, because 'ordinal' comes from a normal Python * unicode string */ *out_value = ordinal; return 1; } + *out_value = 0; /* silence a gcc warning if this function is inlined */ return 0; + + error: + Py_DECREF(io); + *out_value = 0; /* silence a gcc warning if this function is inlined */ + return -1; } static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob) @@ -3982,7 +3984,8 @@ static void dl_dealloc(DynLibObject *dlobj) { - dlclose(dlobj->dl_handle); + if (dlobj->dl_handle != NULL) + dlclose(dlobj->dl_handle); free(dlobj->dl_name); PyObject_Del(dlobj); } @@ -3992,6 +3995,17 @@ return PyText_FromFormat("", dlobj->dl_name); } +static int dl_check_closed(DynLibObject *dlobj) +{ + if (dlobj->dl_handle == NULL) + { + PyErr_Format(PyExc_ValueError, "library '%s' has already been closed", + dlobj->dl_name); + return -1; + } + return 0; +} + static PyObject *dl_load_function(DynLibObject *dlobj, PyObject *args) { CTypeDescrObject *ct; @@ -4002,6 +4016,9 @@ &CTypeDescr_Type, &ct, &funcname)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + if (!(ct->ct_flags & (CT_FUNCTIONPTR | CT_POINTER | CT_ARRAY))) { PyErr_Format(PyExc_TypeError, "function or pointer or array cdata expected, got '%s'", @@ -4034,6 +4051,9 @@ &CTypeDescr_Type, &ct, &varname)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + dlerror(); /* clear error condition */ data = dlsym(dlobj->dl_handle, varname); if (data == NULL) { @@ -4059,6 +4079,9 @@ &CTypeDescr_Type, &ct, &varname, &value)) return NULL; + if (dl_check_closed(dlobj) < 0) + return NULL; + dlerror(); /* clear error condition */ data = dlsym(dlobj->dl_handle, varname); if (data == NULL) { @@ -4074,10 +4097,21 @@ return Py_None; } +static PyObject *dl_close_lib(DynLibObject *dlobj, PyObject *no_args) +{ + if (dl_check_closed(dlobj) < 0) + return NULL; + dlclose(dlobj->dl_handle); + dlobj->dl_handle = NULL; + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef dl_methods[] = { {"load_function", (PyCFunction)dl_load_function, METH_VARARGS}, {"read_variable", (PyCFunction)dl_read_variable, METH_VARARGS}, {"write_variable", (PyCFunction)dl_write_variable, METH_VARARGS}, + {"close_lib", (PyCFunction)dl_close_lib, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -4113,44 +4147,95 @@ dl_methods, /* tp_methods */ }; -static PyObject *b_load_library(PyObject *self, PyObject *args) -{ - char *filename_or_null, *printable_filename; +static void *b_do_dlopen(PyObject *args, char **p_printable_filename, + PyObject **p_temp) +{ + /* Logic to call the correct version of dlopen(). Returns NULL in case of error. + Otherwise, '*p_printable_filename' will point to a printable char version of + the filename (maybe utf-8-encoded). '*p_temp' will be set either to NULL or + to a temporary object that must be freed after looking at printable_filename. + */ void *handle; - DynLibObject *dlobj; + char *filename_or_null; int flags = 0; - + *p_temp = NULL; + if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { PyObject *dummy; if (!PyArg_ParseTuple(args, "|Oi:load_library", &dummy, &flags)) return NULL; filename_or_null = NULL; - } - else if (!PyArg_ParseTuple(args, "et|i:load_library", - Py_FileSystemDefaultEncoding, &filename_or_null, - &flags)) - return NULL; - + *p_printable_filename = ""; + } + else + { + PyObject *s = PyTuple_GET_ITEM(args, 0); +#ifdef MS_WIN32 + Py_UNICODE *filenameW; + if (PyArg_ParseTuple(args, "u|i:load_library", &filenameW, &flags)) + { +#if PY_MAJOR_VERSION < 3 + s = PyUnicode_AsUTF8String(s); + if (s == NULL) + return NULL; + *p_temp = s; +#endif + *p_printable_filename = PyText_AsUTF8(s); + if (*p_printable_filename == NULL) + return NULL; + + handle = dlopenW(filenameW); + goto got_handle; + } + PyErr_Clear(); +#endif + if (!PyArg_ParseTuple(args, "et|i:load_library", + Py_FileSystemDefaultEncoding, &filename_or_null, &flags)) + return NULL; + + *p_printable_filename = PyText_AsUTF8(s); + if (*p_printable_filename == NULL) + return NULL; + } if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) flags |= RTLD_NOW; - printable_filename = filename_or_null ? filename_or_null : ""; handle = dlopen(filename_or_null, flags); + +#ifdef MS_WIN32 + got_handle: +#endif if (handle == NULL) { const char *error = dlerror(); - PyErr_Format(PyExc_OSError, "cannot load library %s: %s", - printable_filename, error); + PyErr_Format(PyExc_OSError, "cannot load library '%s': %s", + *p_printable_filename, error); return NULL; } + return handle; +} + +static PyObject *b_load_library(PyObject *self, PyObject *args) +{ + char *printable_filename; + PyObject *temp; + void *handle; + DynLibObject *dlobj = NULL; + + handle = b_do_dlopen(args, &printable_filename, &temp); + if (handle == NULL) + goto error; dlobj = PyObject_New(DynLibObject, &dl_type); if (dlobj == NULL) { dlclose(handle); - return NULL; + goto error; } dlobj->dl_handle = handle; dlobj->dl_name = strdup(printable_filename); + + error: + Py_XDECREF(temp); return (PyObject *)dlobj; } @@ -4796,7 +4881,6 @@ if (PyText_GetSize(fname) == 0 && ftype->ct_flags & (CT_STRUCT|CT_UNION)) { /* a nested anonymous struct or union */ - /* note: it seems we only get here with ffi.verify() */ CFieldObject *cfsrc = (CFieldObject *)ftype->ct_extra; for (; cfsrc != NULL; cfsrc = cfsrc->cf_next) { /* broken complexity in the call to get_field_name(), diff --git a/c/cdlopen.c b/c/cdlopen.c --- a/c/cdlopen.c +++ b/c/cdlopen.c @@ -40,35 +40,18 @@ static PyObject *ffi_dlopen(PyObject *self, PyObject *args) { - char *filename_or_null, *printable_filename; + char *modname; + PyObject *temp, *result = NULL; void *handle; - int flags = 0; - if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { - PyObject *dummy; - if (!PyArg_ParseTuple(args, "|Oi:load_library", - &dummy, &flags)) - return NULL; - filename_or_null = NULL; + handle = b_do_dlopen(args, &modname, &temp); + if (handle != NULL) + { + result = (PyObject *)lib_internal_new((FFIObject *)self, + modname, handle); } - else if (!PyArg_ParseTuple(args, "et|i:load_library", - Py_FileSystemDefaultEncoding, &filename_or_null, - &flags)) - return NULL; - - if ((flags & (RTLD_NOW | RTLD_LAZY)) == 0) - flags |= RTLD_NOW; - printable_filename = filename_or_null ? filename_or_null : ""; - - handle = dlopen(filename_or_null, flags); - if (handle == NULL) { - const char *error = dlerror(); - PyErr_Format(PyExc_OSError, "cannot load library '%s': %s", - printable_filename, error); - return NULL; - } - return (PyObject *)lib_internal_new((FFIObject *)self, - printable_filename, handle); + Py_XDECREF(temp); + return result; } static PyObject *ffi_dlclose(PyObject *self, PyObject *args) diff --git a/c/misc_win32.h b/c/misc_win32.h --- a/c/misc_win32.h +++ b/c/misc_win32.h @@ -192,7 +192,12 @@ static void *dlopen(const char *filename, int flag) { - return (void *)LoadLibrary(filename); + return (void *)LoadLibraryA(filename); +} + +static void *dlopenW(const wchar_t *filename) +{ + return (void *)LoadLibraryW(filename); } static void *dlsym(void *handle, const char *symbol) diff --git a/c/realize_c_type.c b/c/realize_c_type.c --- a/c/realize_c_type.c +++ b/c/realize_c_type.c @@ -737,13 +737,13 @@ return -1; } - if (fld->field_offset == (size_t)-1) { + if (ctf != NULL && fld->field_offset == (size_t)-1) { /* unnamed struct, with field positions and sizes entirely determined by complete_struct_or_union() and not checked. Or, bitfields (field_size >= 0), similarly not checked. */ assert(fld->field_size == (size_t)-1 || fbitsize >= 0); } - else if (detect_custom_layout(ct, SF_STD_FIELD_POS, + else if (ctf == NULL || detect_custom_layout(ct, SF_STD_FIELD_POS, ctf->ct_size, fld->field_size, "wrong size for field '", fld->name, "'") < 0) { diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -896,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -148,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -169,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -560,7 +560,13 @@ ``NAME.cpython-35m-x86_64-linux-gnu.so``. You can manually rename it to ``NAME.abi3.so``, or use setuptools version 26 or later. Also, note that compiling with a debug version of Python will not actually define -``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy. +``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy. Finally, +``Py_LIMITED_API`` is not defined on Windows, because this makes +modules which cannot be used with ``virtualenv`` (issues `#355`__ and +`#350`__). + +.. __: https://bitbucket.org/cffi/cffi/issues/355/importerror-dll-load-failed-on-windows +.. __: https://bitbucket.org/cffi/cffi/issues/350/issue-with-py_limited_api-on-windows **ffibuilder.compile(tmpdir='.', verbose=False, debug=None):** explicitly generate the .py or .c file, diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -2,6 +2,42 @@ What's New ====================== + +v1.11.5 +======= + +* `Issue #357`_: fix ``ffi.emit_python_code()`` which generated a buggy + Python file if you are using a ``struct`` with an anonymous ``union`` + field or vice-versa. + +* Windows: ``ffi.dlopen()`` should now handle unicode filenames. + +* ABI mode: implemented ``ffi.dlclose()`` for the in-line case (it used + to be present only in the out-of-line case). + +* Fixed a corner case for ``setup.py install --record=xx --root=yy`` + with an out-of-line ABI module. Also fixed `Issue #345`_. + +* More hacks on Windows for running CFFI's own ``setup.py``. + +* `Issue #358`_: in embedding, to protect against (the rare case of) + Python initialization from several threads in parallel, we have to use + a spin-lock. On CPython 3 it is worse because it might spin-lock for + a long time (execution of ``Py_InitializeEx()``). Sadly, recent + changes to CPython make that solution needed on CPython 2 too. + +* CPython 3 on Windows: we no longer compile with ``Py_LIMITED_API`` + by default because such modules cannot be used with virtualenv. + `Issue #350`_ mentions a workaround if you still want that and are not + concerned about virtualenv: pass a ``define_macros=[("Py_LIMITED_API", + None)]`` to the ``ffibuilder.set_source()`` call. + +.. _`Issue #345`: https://bitbucket.org/cffi/cffi/issues/345/ +.. _`Issue #350`: https://bitbucket.org/cffi/cffi/issues/350/ +.. _`Issue #358`: https://bitbucket.org/cffi/cffi/issues/358/ +.. _`Issue #357`: https://bitbucket.org/cffi/cffi/issues/357/ + + v1.11.4 ======= diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ import subprocess import errno +# on Windows we give up and always import setuptools early to fix things for us +if sys.platform == "win32": + import setuptools + sources = ['c/_cffi_backend.c'] libraries = ['ffi'] diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py --- a/testing/cffi0/test_function.py +++ b/testing/cffi0/test_function.py @@ -499,3 +499,23 @@ """) m = ffi.dlopen(lib_m) assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc', 'myvar'] + + def test_dlclose(self): + if self.Backend is CTypesBackend: + py.test.skip("not with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef("int foobar(void); int foobaz;") + lib = ffi.dlopen(lib_m) + ffi.dlclose(lib) + e = py.test.raises(ValueError, ffi.dlclose, lib) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobar') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobaz') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py --- a/testing/cffi0/test_ownlib.py +++ b/testing/cffi0/test_ownlib.py @@ -114,8 +114,12 @@ if sys.platform == 'win32': import os # did we already build it? - if os.path.exists(str(udir.join('testownlib.dll'))): - cls.module = str(udir.join('testownlib.dll')) + if cls.Backend is CTypesBackend: + dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend + else: + dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') # non-ascii char + if os.path.exists(dll_path): + cls.module = dll_path return # try (not too hard) to find the version used to compile this python # no mingw @@ -135,8 +139,9 @@ if os.path.isfile(vcvarsall): cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ ' /LD /Fetestownlib.dll' - subprocess.check_call(cmd, cwd = str(udir), shell=True) - cls.module = str(udir.join('testownlib.dll')) + subprocess.check_call(cmd, cwd = str(udir), shell=True) + os.rename(str(udir) + '\\testownlib.dll', dll_path) + cls.module = dll_path else: subprocess.check_call( 'cc testownlib.c -shared -fPIC -o testownlib.so', diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py --- a/testing/cffi1/test_re_python.py +++ b/testing/cffi1/test_re_python.py @@ -1,8 +1,9 @@ -import sys +import sys, os import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing from testing.udir import udir +from testing.support import u def setup_module(mod): @@ -35,6 +36,13 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + if sys.platform == "win32": + # test with a non-ascii char + outputfn1 = outputfilename + ofn, oext = os.path.splitext(outputfn1) + outputfilename = ofn + (u+'\u03be') + oext + #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) + os.rename(outputfn1, outputfilename) mod.extmod = outputfilename mod.tmpdir = tmpdir # @@ -55,6 +63,9 @@ typedef struct bar_s { int x; signed char a[]; } bar_t; enum foo_e { AA, BB, CC }; int strlen(const char *); + struct with_union { union { int a; char b; }; }; + union with_struct { struct { int a; char b; }; }; + struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; }; """) ffi.set_source('re_python_pysrc', None) ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py'))) @@ -104,12 +115,16 @@ from re_python_pysrc import ffi lib = ffi.dlopen(extmod) ffi.dlclose(lib) + if type(extmod) is not str: # unicode, on python 2 + str_extmod = extmod.encode('utf-8') + else: + str_extmod = extmod e = py.test.raises(ffi.error, ffi.dlclose, lib) assert str(e.value).startswith( - "library '%s' is already closed" % (extmod,)) + "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( - "library '%s' has been closed" % (extmod,)) + "library '%s' has been closed" % (str_extmod,)) def test_constant_via_lib(): from re_python_pysrc import ffi @@ -212,3 +227,23 @@ ffi.set_source('test_partial_enum', None) py.test.raises(VerificationMissing, ffi.emit_python_code, str(tmpdir.join('test_partial_enum.py'))) + +def test_anonymous_union_inside_struct(): + # based on issue #357 + from re_python_pysrc import ffi + INT = ffi.sizeof("int") + assert ffi.offsetof("struct with_union", "a") == 0 + assert ffi.offsetof("struct with_union", "b") == 0 + assert ffi.sizeof("struct with_union") == INT + # + assert ffi.offsetof("union with_struct", "a") == 0 + assert ffi.offsetof("union with_struct", "b") == INT + assert ffi.sizeof("union with_struct") >= INT + 1 + # + FLOAT = ffi.sizeof("float") + assert ffi.sizeof("struct NVGcolor") == FLOAT * 4 + assert ffi.offsetof("struct NVGcolor", "rgba") == 0 + assert ffi.offsetof("struct NVGcolor", "r") == 0 + assert ffi.offsetof("struct NVGcolor", "g") == FLOAT + assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2 + assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3 diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2297,3 +2297,11 @@ else: assert lib.__loader__ is None assert lib.__spec__ is None + +def test_realize_struct_error(): + ffi = FFI() + ffi.cdef("""typedef ... foo_t; struct foo_s { void (*x)(foo_t); };""") + lib = verify(ffi, "test_realize_struct_error", """ + typedef int foo_t; struct foo_s { void (*x)(foo_t); }; + """) + py.test.raises(TypeError, ffi.new, "struct foo_s *") From pypy.commits at gmail.com Tue Feb 27 13:15:25 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:15:25 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: Bump to 1.11.5 Message-ID: <5a95a03d.87a4df0a.5a6df.1ddf@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3109:bad3cd7621e6 Date: 2018-02-27 19:15 +0100 http://bitbucket.org/cffi/cffi/changeset/bad3cd7621e6/ Log: Bump to 1.11.5 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.4" +#define CFFI_VERSION "1.11.5" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.5", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.4' +release = '1.11.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.4: +* Checksums of the "source" package version 1.11.5: - - MD5: 8ef0f852498908b20bc508799901fe93 + - MD5: ... - - SHA: d0c4f7d4b6af4c438a17b2f2334f2aa5ff7cc4db + - SHA: ... - - SHA256: df9083a992b17a28cd4251a3f5c879e0198bb26c9e808c4647e0a18739f1d11d + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -190,7 +190,7 @@ `Mailing list `_ """, - version='1.11.4', + version='1.11.5', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Tue Feb 27 13:17:42 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:17:42 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: Added tag v1.11.5 for changeset 48416163071e Message-ID: <5a95a0c6.92aa1c0a.7c513.0efd@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3111:7e2fa5b951e0 Date: 2018-02-27 19:17 +0100 http://bitbucket.org/cffi/cffi/changeset/7e2fa5b951e0/ Log: Added tag v1.11.5 for changeset 48416163071e diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,4 @@ 5f9690f5832b0292056df45f72314d69c191f75a v1.11.2 1aafccb9255dbb36f8e785b65624e39628cee63a v1.11.3 e08abd4703fef26f036e82255f4070277a9e03bd v1.11.4 +48416163071ed48300c3ae4358cc7fd841912413 v1.11.5 From pypy.commits at gmail.com Tue Feb 27 13:17:40 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:17:40 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: md5/sha Message-ID: <5a95a0c4.0fe71c0a.2256c.0d8a@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3110:48416163071e Date: 2018-02-27 19:17 +0100 http://bitbucket.org/cffi/cffi/changeset/48416163071e/ Log: md5/sha diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,11 +55,11 @@ * Checksums of the "source" package version 1.11.5: - - MD5: ... + - MD5: ac8492f4ad952360737413e82d661908 - - SHA: ... + - SHA: 1686e6689a691414d3d22626c837adeee3996dd9 - - SHA256: ... + - SHA256: e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From pypy.commits at gmail.com Tue Feb 27 13:17:45 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:17:45 -0800 (PST) Subject: [pypy-commit] cffi default: hg merge release-1.11 Message-ID: <5a95a0c9.8c941c0a.597dd.0ce3@mx.google.com> Author: Armin Rigo Branch: Changeset: r3112:70790d813156 Date: 2018-02-27 19:17 +0100 http://bitbucket.org/cffi/cffi/changeset/70790d813156/ Log: hg merge release-1.11 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,4 @@ 5f9690f5832b0292056df45f72314d69c191f75a v1.11.2 1aafccb9255dbb36f8e785b65624e39628cee63a v1.11.3 e08abd4703fef26f036e82255f4070277a9e03bd v1.11.4 +48416163071ed48300c3ae4358cc7fd841912413 v1.11.5 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.4" +#define CFFI_VERSION "1.11.5" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.5", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.4' +release = '1.11.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.4: +* Checksums of the "source" package version 1.11.5: - - MD5: 8ef0f852498908b20bc508799901fe93 + - MD5: ac8492f4ad952360737413e82d661908 - - SHA: d0c4f7d4b6af4c438a17b2f2334f2aa5ff7cc4db + - SHA: 1686e6689a691414d3d22626c837adeee3996dd9 - - SHA256: df9083a992b17a28cd4251a3f5c879e0198bb26c9e808c4647e0a18739f1d11d + - SHA256: e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -190,7 +190,7 @@ `Mailing list `_ """, - version='1.11.4', + version='1.11.5', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Tue Feb 27 13:57:08 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:57:08 -0800 (PST) Subject: [pypy-commit] cffi default: Direct tests for close_lib() Message-ID: <5a95aa04.8b8edf0a.7b297.da7e@mx.google.com> Author: Armin Rigo Branch: Changeset: r3113:191ee1e092eb Date: 2018-02-27 19:56 +0100 http://bitbucket.org/cffi/cffi/changeset/191ee1e092eb/ Log: Direct tests for close_lib() diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -406,6 +406,9 @@ # the next one is from 'libm', not 'libc', but we assume # that it is already loaded too, so it should work assert x.load_function(BVoidP, 'sqrt') + # + x.close_lib() + py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') def test_no_len_on_nonarray(): p = new_primitive_type("int") @@ -1221,6 +1224,9 @@ ll = find_and_load_library('c') stderr = ll.read_variable(BVoidP, "stderr") assert stderr == cast(BVoidP, _testfunc(8)) + # + ll.close_lib() + py.test.raises(ValueError, ll.read_variable, BVoidP, "stderr") def test_read_variable_as_unknown_length_array(): ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard @@ -1247,6 +1253,9 @@ assert not ll.read_variable(BVoidP, "stderr") ll.write_variable(BVoidP, "stderr", stderr) assert ll.read_variable(BVoidP, "stderr") == stderr + # + ll.close_lib() + py.test.raises(ValueError, ll.write_variable, BVoidP, "stderr", stderr) def test_callback(): BInt = new_primitive_type("int") From pypy.commits at gmail.com Tue Feb 27 13:59:22 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:59:22 -0800 (PST) Subject: [pypy-commit] pypy default: Update the _cffi_backend module to 1.11.5 Message-ID: <5a95aa8a.85261c0a.1845e.1e20@mx.google.com> Author: Armin Rigo Branch: Changeset: r93898:5e00d00261f8 Date: 2018-02-27 19:58 +0100 http://bitbucket.org/pypy/pypy/changeset/5e00d00261f8/ Log: Update the _cffi_backend module to 1.11.5 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.4" +VERSION = "1.11.5" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -38,9 +38,16 @@ space = self.space return space.newtext("" % self.name) + def check_closed(self): + if self.handle == rffi.cast(DLLHANDLE, 0): + raise oefmt(space.w_ValueError, + "library '%s' has already been closed", + self.name) + @unwrap_spec(w_ctype=W_CType, name='text') def load_function(self, w_ctype, name): from pypy.module._cffi_backend import ctypeptr, ctypearray + self.check_closed() space = self.space # if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray): @@ -60,6 +67,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def read_variable(self, w_ctype, name): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -71,6 +79,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def write_variable(self, w_ctype, name, w_value): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -80,6 +89,10 @@ name, self.name) w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value) + def close_lib(self): + self.check_closed() + self._finalize_() + W_Library.typedef = TypeDef( '_cffi_backend.Library', @@ -87,6 +100,7 @@ load_function = interp2app(W_Library.load_function), read_variable = interp2app(W_Library.read_variable), write_variable = interp2app(W_Library.write_variable), + close_lib = interp2app(W_Library.close_lib), ) W_Library.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.5", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): From pypy.commits at gmail.com Tue Feb 27 13:59:20 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 10:59:20 -0800 (PST) Subject: [pypy-commit] pypy default: import cffi/70790d813156 (1.11.5) Message-ID: <5a95aa88.7499df0a.14ede.ac57@mx.google.com> Author: Armin Rigo Branch: Changeset: r93897:f80f173700c8 Date: 2018-02-27 19:45 +0100 http://bitbucket.org/pypy/pypy/changeset/f80f173700c8/ Log: import cffi/70790d813156 (1.11.5) diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -896,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -143,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -164,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -500,3 +500,23 @@ """) m = ffi.dlopen(lib_m) assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc', 'myvar'] + + def test_dlclose(self): + if self.Backend is CTypesBackend: + py.test.skip("not with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef("int foobar(void); int foobaz;") + lib = ffi.dlopen(lib_m) + ffi.dlclose(lib) + e = py.test.raises(ValueError, ffi.dlclose, lib) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobar') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobaz') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py @@ -115,8 +115,12 @@ if sys.platform == 'win32': import os # did we already build it? - if os.path.exists(str(udir.join('testownlib.dll'))): - cls.module = str(udir.join('testownlib.dll')) + if cls.Backend is CTypesBackend: + dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend + else: + dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') # non-ascii char + if os.path.exists(dll_path): + cls.module = dll_path return # try (not too hard) to find the version used to compile this python # no mingw @@ -136,8 +140,9 @@ if os.path.isfile(vcvarsall): cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ ' /LD /Fetestownlib.dll' - subprocess.check_call(cmd, cwd = str(udir), shell=True) - cls.module = str(udir.join('testownlib.dll')) + subprocess.check_call(cmd, cwd = str(udir), shell=True) + os.rename(str(udir) + '\\testownlib.dll', dll_path) + cls.module = dll_path else: subprocess.check_call( 'cc testownlib.c -shared -fPIC -o testownlib.so', diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py @@ -1,9 +1,10 @@ # Generated by pypy/tool/import_cffi.py -import sys +import sys, os import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from pypy.module.test_lib_pypy.cffi_tests.support import u def setup_module(mod): @@ -36,6 +37,13 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + if sys.platform == "win32": + # test with a non-ascii char + outputfn1 = outputfilename + ofn, oext = os.path.splitext(outputfn1) + outputfilename = ofn + (u+'\u03be') + oext + #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) + os.rename(outputfn1, outputfilename) mod.extmod = outputfilename mod.tmpdir = tmpdir # @@ -56,6 +64,9 @@ typedef struct bar_s { int x; signed char a[]; } bar_t; enum foo_e { AA, BB, CC }; int strlen(const char *); + struct with_union { union { int a; char b; }; }; + union with_struct { struct { int a; char b; }; }; + struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; }; """) ffi.set_source('re_python_pysrc', None) ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py'))) @@ -105,12 +116,16 @@ from re_python_pysrc import ffi lib = ffi.dlopen(extmod) ffi.dlclose(lib) + if type(extmod) is not str: # unicode, on python 2 + str_extmod = extmod.encode('utf-8') + else: + str_extmod = extmod e = py.test.raises(ffi.error, ffi.dlclose, lib) assert str(e.value).startswith( - "library '%s' is already closed" % (extmod,)) + "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( - "library '%s' has been closed" % (extmod,)) + "library '%s' has been closed" % (str_extmod,)) def test_constant_via_lib(): from re_python_pysrc import ffi @@ -213,3 +228,23 @@ ffi.set_source('test_partial_enum', None) py.test.raises(VerificationMissing, ffi.emit_python_code, str(tmpdir.join('test_partial_enum.py'))) + +def test_anonymous_union_inside_struct(): + # based on issue #357 + from re_python_pysrc import ffi + INT = ffi.sizeof("int") + assert ffi.offsetof("struct with_union", "a") == 0 + assert ffi.offsetof("struct with_union", "b") == 0 + assert ffi.sizeof("struct with_union") == INT + # + assert ffi.offsetof("union with_struct", "a") == 0 + assert ffi.offsetof("union with_struct", "b") == INT + assert ffi.sizeof("union with_struct") >= INT + 1 + # + FLOAT = ffi.sizeof("float") + assert ffi.sizeof("struct NVGcolor") == FLOAT * 4 + assert ffi.offsetof("struct NVGcolor", "rgba") == 0 + assert ffi.offsetof("struct NVGcolor", "r") == 0 + assert ffi.offsetof("struct NVGcolor", "g") == FLOAT + assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2 + assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3 From pypy.commits at gmail.com Tue Feb 27 14:04:05 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 11:04:05 -0800 (PST) Subject: [pypy-commit] pypy default: Oops Message-ID: <5a95aba5.87a0df0a.17c9c.d0b7@mx.google.com> Author: Armin Rigo Branch: Changeset: r93899:98de07e737f3 Date: 2018-02-27 20:02 +0100 http://bitbucket.org/pypy/pypy/changeset/98de07e737f3/ Log: Oops diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.4 +Version: 1.11.5 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -40,7 +40,7 @@ def check_closed(self): if self.handle == rffi.cast(DLLHANDLE, 0): - raise oefmt(space.w_ValueError, + raise oefmt(self.space.w_ValueError, "library '%s' has already been closed", self.name) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -395,6 +395,9 @@ # the next one is from 'libm', not 'libc', but we assume # that it is already loaded too, so it should work assert x.load_function(BVoidP, 'sqrt') + # + x.close_lib() + py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') def test_no_len_on_nonarray(): p = new_primitive_type("int") @@ -1210,6 +1213,9 @@ ll = find_and_load_library('c') stderr = ll.read_variable(BVoidP, "stderr") assert stderr == cast(BVoidP, _testfunc(8)) + # + ll.close_lib() + py.test.raises(ValueError, ll.read_variable, BVoidP, "stderr") def test_read_variable_as_unknown_length_array(): ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard @@ -1236,6 +1242,9 @@ assert not ll.read_variable(BVoidP, "stderr") ll.write_variable(BVoidP, "stderr", stderr) assert ll.read_variable(BVoidP, "stderr") == stderr + # + ll.close_lib() + py.test.raises(ValueError, ll.write_variable, BVoidP, "stderr", stderr) def test_callback(): BInt = new_primitive_type("int") From pypy.commits at gmail.com Tue Feb 27 15:20:30 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Feb 2018 12:20:30 -0800 (PST) Subject: [pypy-commit] cffi default: Ignore multiple dlclose(), like file.close() does in Python Message-ID: <5a95bd8e.c7581c0a.cfe47.3a76@mx.google.com> Author: Armin Rigo Branch: Changeset: r3114:99940f1f5402 Date: 2018-02-27 21:18 +0100 http://bitbucket.org/cffi/cffi/changeset/99940f1f5402/ Log: Ignore multiple dlclose(), like file.close() does in Python diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4099,10 +4099,11 @@ static PyObject *dl_close_lib(DynLibObject *dlobj, PyObject *no_args) { - if (dl_check_closed(dlobj) < 0) - return NULL; - dlclose(dlobj->dl_handle); - dlobj->dl_handle = NULL; + if (dlobj->dl_handle != NULL) + { + dlclose(dlobj->dl_handle); + dlobj->dl_handle = NULL; + } Py_INCREF(Py_None); return Py_None; } diff --git a/c/cdlopen.c b/c/cdlopen.c --- a/c/cdlopen.c +++ b/c/cdlopen.c @@ -62,22 +62,17 @@ return NULL; libhandle = lib->l_libhandle; - lib->l_libhandle = NULL; + if (libhandle != NULL) + { + lib->l_libhandle = NULL; - if (libhandle == NULL) { - PyErr_Format(FFIError, "library '%s' is already closed " - "or was not created with ffi.dlopen()", - PyText_AS_UTF8(lib->l_libname)); - return NULL; + /* Clear the dict to force further accesses to do cdlopen_fetch() + again, and fail because the library was closed. */ + PyDict_Clear(lib->l_dict); + + if (cdlopen_close(lib->l_libname, libhandle) < 0) + return NULL; } - - /* Clear the dict to force further accesses to do cdlopen_fetch() - again, and fail because the library was closed. */ - PyDict_Clear(lib->l_dict); - - if (cdlopen_close(lib->l_libname, libhandle) < 0) - return NULL; - Py_INCREF(Py_None); return Py_None; } diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -409,6 +409,7 @@ # x.close_lib() py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py --- a/testing/cffi0/test_function.py +++ b/testing/cffi0/test_function.py @@ -507,9 +507,6 @@ ffi.cdef("int foobar(void); int foobaz;") lib = ffi.dlopen(lib_m) ffi.dlclose(lib) - e = py.test.raises(ValueError, ffi.dlclose, lib) - assert str(e.value).startswith("library '") - assert str(e.value).endswith("' has already been closed") e = py.test.raises(ValueError, getattr, lib, 'foobar') assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") @@ -519,3 +516,4 @@ e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") + ffi.dlclose(lib) # does not raise diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py --- a/testing/cffi1/test_re_python.py +++ b/testing/cffi1/test_re_python.py @@ -119,12 +119,10 @@ str_extmod = extmod.encode('utf-8') else: str_extmod = extmod - e = py.test.raises(ffi.error, ffi.dlclose, lib) - assert str(e.value).startswith( - "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (str_extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(): from re_python_pysrc import ffi From pypy.commits at gmail.com Tue Feb 27 18:00:24 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 27 Feb 2018 15:00:24 -0800 (PST) Subject: [pypy-commit] pypy default: Do overwrite existing slot in fill_slot(): Message-ID: <5a95e308.6980df0a.a70a7.1473@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93900:276b2e484fdf Date: 2018-02-27 22:59 +0000 http://bitbucket.org/pypy/pypy/changeset/276b2e484fdf/ Log: Do overwrite existing slot in fill_slot(): otherwise subclasses can't overrride their parents' slots. Reverts e61e2f4a32fa Fixes issue #2760 diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -648,6 +648,33 @@ self.attr1 = 123 assert module.test_tp_getattro(C(), 123) + def test_issue_2760_getattr(self): + module = self.import_extension('foo', [ + ("get_foo", "METH_O", + ''' + char* name = "foo"; + PyTypeObject *tp = Py_TYPE(args); + PyObject *res; + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(args, name); + } + else if (tp->tp_getattro != NULL) { + PyObject *w = PyString_FromString(name); + res = (*tp->tp_getattro)(args, w); + Py_DECREF(w); + } + else { + res = Py_None; + } + return res; + ''')]) + class Passthrough(object): + def __getattr__(self, name): + return name + + obj = Passthrough() + assert module.get_foo(obj) == 'foo' + def test_nb_int(self): module = self.import_extension('foo', [ ("nb_int", "METH_VARARGS", diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -236,11 +236,6 @@ def update_all_slots(space, w_type, pto): # fill slots in pto - # Not very sure about it, but according to - # test_call_tp_dealloc, we should not - # overwrite slots that are already set: these ones are probably - # coming from a parent C type. - for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: slot_func_helper = None w_descr = w_type.dict_w.get(method_name, None) @@ -276,8 +271,7 @@ def fill_slot(space, pto, w_type, slot_names, slot_func_helper): # XXX special case wrapper-functions and use a "specific" slot func if len(slot_names) == 1: - if not getattr(pto, slot_names[0]): - setattr(pto, slot_names[0], slot_func_helper) + setattr(pto, slot_names[0], slot_func_helper) elif ((w_type is space.w_list or w_type is space.w_tuple) and slot_names[0] == 'c_tp_as_number'): # XXX hack - how can we generalize this? The problem is method @@ -311,8 +305,7 @@ struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True) setattr(pto, slot_names[0], struct) - if not getattr(struct, slot_names[1]): - setattr(struct, slot_names[1], slot_func_helper) + setattr(struct, slot_names[1], slot_func_helper) def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented From pypy.commits at gmail.com Wed Feb 28 09:58:47 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Feb 2018 06:58:47 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: fix Message-ID: <5a96c3a7.4b7e1c0a.b0637.c2c9@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93904:02f016f93ecf Date: 2018-02-28 15:57 +0100 http://bitbucket.org/pypy/pypy/changeset/02f016f93ecf/ Log: fix diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -949,7 +949,7 @@ else: resumedescr = ResumeGuardExcDescr() elif opnum == rop.GUARD_COMPATIBLE: - assert not copied_guard # XXX for now? + assert copied_from_descr is None # XXX for now? resumedescr = GuardCompatibleDescr() else: if copied_from_descr is not None: From pypy.commits at gmail.com Wed Feb 28 10:21:37 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Feb 2018 07:21:37 -0800 (PST) Subject: [pypy-commit] pypy default: silence a GCC int-in-bool-context warning Message-ID: <5a96c901.51811c0a.e7081.105d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93905:8e52e1e16d7c Date: 2018-02-28 16:10 +0100 http://bitbucket.org/pypy/pypy/changeset/8e52e1e16d7c/ Log: silence a GCC int-in-bool-context warning this is equivalent to the fix of CPython issue https://bugs.python.org/issue31474 diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h --- a/pypy/module/cpyext/include/pymem.h +++ b/pypy/module/cpyext/include/pymem.h @@ -7,8 +7,8 @@ extern "C" { #endif -#define PyMem_MALLOC(n) malloc((n) ? (n) : 1) -#define PyMem_REALLOC(p, n) realloc((p), (n) ? (n) : 1) +#define PyMem_MALLOC(n) malloc(((n) != 0) ? (n) : 1) +#define PyMem_REALLOC(p, n) realloc((p), ((n) != 0) ? (n) : 1) #define PyMem_FREE free PyAPI_FUNC(void *) PyMem_Malloc(size_t); From pypy.commits at gmail.com Wed Feb 28 09:58:45 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Feb 2018 06:58:45 -0800 (PST) Subject: [pypy-commit] pypy guard-compatible: merge default Message-ID: <5a96c3a5.c4d51c0a.b48c.0ee3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r93903:047c33d38a20 Date: 2018-02-22 20:33 +0100 http://bitbucket.org/pypy/pypy/changeset/047c33d38a20/ Log: merge default diff too long, truncating to 2000 out of 10294 lines diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + + at pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.3 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,37 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -17,10 +17,13 @@ """ from __future__ import division -import time as _time +import time as _timemodule import math as _math import struct as _struct +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + _SENTINEL = object() def _cmp(x, y): @@ -179,7 +182,7 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. @@ -247,7 +250,7 @@ else: push(ch) newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) + return _timemodule.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): @@ -433,7 +436,7 @@ raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) -class timedelta(object): +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -489,7 +492,7 @@ if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -667,7 +670,7 @@ timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) -class date(object): +class date(dateinterop): """Concrete date type. Constructors: @@ -707,12 +710,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -724,13 +727,13 @@ @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t) @classmethod @@ -1061,7 +1064,7 @@ _tzinfo_class = tzinfo -class time(object): +class time(timeinterop): """Time with time zone. Constructors: @@ -1097,14 +1100,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1408,15 +1411,20 @@ if isinstance(year, bytes) and len(year) == 10 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1461,7 +1469,7 @@ A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - converter = _time.localtime if tz is None else _time.gmtime + converter = _timemodule.localtime if tz is None else _timemodule.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) @@ -1470,7 +1478,7 @@ @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - return cls._from_timestamp(_time.gmtime, t, None) + return cls._from_timestamp(_timemodule.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): @@ -1488,18 +1496,18 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." - t = _time.time() + t = _timemodule.time() return cls.utcfromtimestamp(t) @classmethod @@ -1797,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1805,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -157,7 +157,14 @@ def open(filename, flag='r', mode=0666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") openflag = 0 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,37 @@ Big refactoring of some cpyext code, which avoids a lot of nonsense when calling C from Python and vice-versa: the result is a big speedup in function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import we_are_translated from pypy.interpreter.astcompiler import ast, misc, symtable @@ -266,7 +266,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -276,9 +276,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1096,17 +1096,21 @@ encoding = self.compile_info.encoding flags = self.compile_info.flags unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - try: - sub_strings_w = [parsestring.parsestr(space, encoding, atom_node.get_child(i).get_value(), - unicode_literals) - for i in range(atom_node.num_children())] - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise self.error('(unicode error) %s' % errmsg, atom_node) + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), + unicode_literals)) + except error.OperationError as e: + if not e.match(space, space.w_UnicodeError): + raise + # UnicodeError in literal: turn into SyntaxError + e.normalize_exception(space) + errmsg = space.text_w(space.str(e.get_w_value(space))) + if child is None: + child = atom_node + raise self.error('(unicode error) %s' % errmsg, child) # This implements implicit string concatenation. if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1250,3 +1250,11 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-2: truncated \\xXX escape") + + def test_decode_error_in_string_literal_correct_line(self): + input = "u'a' u'b'\\\n u'c' u'\\x'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-1: truncated \\xXX escape") + assert exc.lineno == 2 + assert exc.offset == 6 diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -210,6 +210,21 @@ def _set_mapdict_storage_and_map(self, storage, map): raise NotImplementedError + + # ------------------------------------------------------------------- + # cpyext support + # these functions will only be seen by the annotator if we translate + # with the cpyext module + + def _cpyext_as_pyobj(self, space): + from pypy.module.cpyext.pyobject import w_root_as_pyobj + return w_root_as_pyobj(self, space) + + def _cpyext_attach_pyobj(self, space, py_obj): + from pypy.module.cpyext.pyobject import w_root_attach_pyobj + return w_root_attach_pyobj(self, space, py_obj) + + # ------------------------------------------------------------------- def is_w(self, space, w_other): diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,5 +1,6 @@ import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be) @@ -26,7 +27,10 @@ got = decode_utf8(space, "\xed\xa0\x80\xed\xb0\x80") assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] @pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) def test_utf32_surrogates(unich): diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -2,11 +2,13 @@ Interp-level implementation of the basic space operations. """ +import math + from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from rpython.rlib.runicode import UNICHR -from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away +from rpython.rlib.rfloat import isfinite, round_double, round_away from rpython.rlib import rfloat import __builtin__ @@ -151,7 +153,7 @@ else: # finite x, and ndigits is not unreasonably large z = round_double(number, ndigits) - if isinf(z): + if math.isinf(z): raise oefmt(space.w_OverflowError, "rounded value too large to represent") return space.newfloat(z) diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -58,6 +58,14 @@ } +class PyPyDateTime(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + } + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -79,7 +87,6 @@ 'get_hidden_tb' : 'interp_magic.get_hidden_tb', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', @@ -108,6 +115,7 @@ "thread": ThreadModule, "intop": IntOpModule, "os": OsModule, + '_pypydatetime': PyPyDateTime, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -107,14 +107,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - def get_console_cp(space): from rpython.rlib import rwin32 # Windows only return space.newtuple([ @@ -142,7 +134,7 @@ space.newbool(debug)) @unwrap_spec(estimate=int) -def add_memory_pressure(estimate): +def add_memory_pressure(space, estimate): """ Add memory pressure of estimate bytes. Useful when calling a C function that internally allocates a big chunk of memory. This instructs the GC to garbage collect sooner than it would otherwise.""" diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -0,0 +1,24 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from rpython.tool.sourcetools import func_with_new_name + +def create_class(name): + class W_Class(W_Root): + 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + def descr_new__(space, w_type): + return space.allocate_instance(W_Class, w_type) + + W_Class.typedef = TypeDef(name, + __new__ = interp2app(func_with_new_name( + W_Class.descr_new__.im_func, + '%s_new' % (name,))), + ) + W_Class.typedef.acceptable_as_base_class = True + return W_Class + +W_DateTime_Time = create_class('pypydatetime_time') +W_DateTime_Date = create_class('pypydatetime_date') +W_DateTime_Delta = create_class('pypydatetime_delta') + + diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.3" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -21,13 +21,13 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True, - add_memory_pressure=True) + flavor='raw', zero=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False, - add_memory_pressure=True) - return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + flavor='raw', zero=False) + w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length) + rgc.add_memory_pressure(datasize, w_res) + return w_res else: w_raw_cdata = space.call_function(self.w_alloc, space.newint(datasize)) @@ -53,7 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) - rgc.add_memory_pressure(datasize) + rgc.add_memory_pressure(datasize, res) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -447,7 +447,10 @@ with self as ptr: w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor) if size != 0: - rgc.add_memory_pressure(size) + if isinstance(w_res, W_CDataGCP): + rgc.add_memory_pressure(size, w_res) + else: + rgc.add_memory_pressure(size, self) return w_res def unpack(self, length): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -538,11 +538,16 @@ assert '\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = "a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 def test_badhandler(self): import codecs diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -61,7 +61,8 @@ ctx = ropenssl.EVP_MD_CTX_new() if ctx is None: raise MemoryError - rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size, + self) try: if copy_from: if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from): diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st + from hypothesis import given, strategies as st, settings except ImportError: pytest.skip("hypothesis required") import os @@ -29,6 +29,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) + at settings(deadline=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1316,8 +1316,9 @@ if not ctx: raise ssl_error(space, "failed to allocate SSL context") - rgc.add_memory_pressure(10 * 1024 * 1024) self = space.allocate_instance(_SSLContext, w_subtype) + assert isinstance(self, _SSLContext) + rgc.add_memory_pressure(10 * 1024 * 1024, self) self.ctx = ctx self.check_hostname = False self.register_finalizer(space) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1090,12 +1090,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -267,6 +267,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 diff --git a/pypy/module/cmath/test/test_cmath.py b/pypy/module/cmath/test/test_cmath.py --- a/pypy/module/cmath/test/test_cmath.py +++ b/pypy/module/cmath/test/test_cmath.py @@ -1,5 +1,4 @@ from __future__ import with_statement -from rpython.rlib.rfloat import copysign, isnan, isinf from pypy.module.cmath import interp_cmath import os, sys, math @@ -11,7 +10,7 @@ assert isinstance(sqrt_special_values[5][1], tuple) assert sqrt_special_values[5][1][0] == 1e200 * 1e200 assert sqrt_special_values[5][1][1] == -0. - assert copysign(1., sqrt_special_values[5][1][1]) == -1. + assert math.copysign(1., sqrt_special_values[5][1][1]) == -1. class AppTestCMath: @@ -142,12 +141,12 @@ """ # special values testing - if isnan(a): - if isnan(b): + if math.isnan(a): + if math.isnan(b): return raise AssertionError(msg + '%r should be nan' % (b,)) - if isinf(a): + if math.isinf(a): if a == b: return raise AssertionError(msg + 'finite result where infinity expected: ' @@ -158,8 +157,7 @@ # and b to have opposite signs; in practice these hardly ever # occur). if not a and not b: - # only check it if we are running on top of CPython >= 2.6 - if sys.version_info >= (2, 6) and copysign(1., a) != copysign(1., b): + if math.copysign(1., a) != math.copysign(1., b): raise AssertionError(msg + 'zero has wrong sign: expected %r, ' 'got %r' % (a, b)) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -96,25 +96,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -1,43 +1,29 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.annlowlevel import llhelper -from pypy.module.cpyext.pyobject import PyObject, make_ref +from rpython.rlib.rarithmetic import widen +from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError +from pypy.module.__pypy__.interp_pypydatetime import (W_DateTime_Date, + W_DateTime_Time, W_DateTime_Delta) from rpython.tool.sourcetools import func_renamer -# API import function +cts.parse_header(parse_dir / 'cpyext_datetime.h') -PyDateTime_CAPI = cpython_struct( - 'PyDateTime_CAPI', - (('DateType', PyTypeObjectPtr), - ('DateTimeType', PyTypeObjectPtr), - ('TimeType', PyTypeObjectPtr), - ('DeltaType', PyTypeObjectPtr), - ('TZInfoType', PyTypeObjectPtr), - ('Date_FromDate', lltype.Ptr(lltype.FuncType( - [rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr], - PyObject))), - ('Time_FromTime', lltype.Ptr(lltype.FuncType( - [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, - PyObject, PyTypeObjectPtr], - PyObject))), - ('DateTime_FromDateAndTime', lltype.Ptr(lltype.FuncType( - [rffi.INT_real, rffi.INT_real, rffi.INT_real, - rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, - PyObject, PyTypeObjectPtr], - PyObject))), - ('Delta_FromDelta', lltype.Ptr(lltype.FuncType( - [rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real, - PyTypeObjectPtr], - PyObject))), - )) +PyDateTime_CAPI = cts.gettype('PyDateTime_CAPI') + +datetimeAPI_global = [] @cpython_api([], lltype.Ptr(PyDateTime_CAPI)) def _PyDateTime_Import(space): + if len(datetimeAPI_global) >0: + return datetimeAPI_global[0] datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) @@ -46,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -76,21 +66,13 @@ _PyDelta_FromDelta.api_func.functype, _PyDelta_FromDelta.api_func.get_wrapper(space)) + datetimeAPI_global.append(datetimeAPI) return datetimeAPI -PyDateTime_DateStruct = lltype.ForwardReference() -PyDateTime_TimeStruct = lltype.ForwardReference() -PyDateTime_DateTimeStruct = lltype.ForwardReference() -cpython_struct("PyDateTime_Date", PyObjectFields, PyDateTime_DateStruct) -PyDateTime_Date = lltype.Ptr(PyDateTime_DateStruct) -cpython_struct("PyDateTime_Time", PyObjectFields, PyDateTime_TimeStruct) -PyDateTime_Time = lltype.Ptr(PyDateTime_TimeStruct) -cpython_struct("PyDateTime_DateTime", PyObjectFields, PyDateTime_DateTimeStruct) -PyDateTime_DateTime = lltype.Ptr(PyDateTime_DateTimeStruct) - -PyDeltaObjectStruct = lltype.ForwardReference() -cpython_struct("PyDateTime_Delta", PyObjectFields, PyDeltaObjectStruct) -PyDateTime_Delta = lltype.Ptr(PyDeltaObjectStruct) +PyDateTime_Time = cts.gettype('PyDateTime_Time*') +PyDateTime_DateTime = cts.gettype('PyDateTime_DateTime*') +PyDateTime_Date = cts.gettype('PyDateTime_Date*') +PyDateTime_Delta = cts.gettype('PyDateTime_Delta*') # Check functions @@ -129,6 +111,75 @@ PyTZInfo_Check, PyTZInfo_CheckExact = make_check_function( "PyTZInfo_Check", "tzinfo") + at bootstrap_function +def init_datetime(space): + # no realize functions since there are no getters + make_typedescr(W_DateTime_Time.typedef, + basestruct=PyDateTime_Time.TO, + attach=type_attach, + dealloc=type_dealloc, + ) + + # why do we need date_dealloc? Since W_DateTime_Date is the base class for + # app level datetime.date. If a c-extension class uses datetime.date for its + # base class and defines a tp_dealloc, we will get this: + # c_class->tp_dealloc == tp_dealloc_func + # c_class->tp_base == datetime.date, + # datetime.date->tp_dealloc = _PyPy_subtype_dealloc + # datetime.date->tp_base = W_DateTime_Date + # W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc + # but _PyPy_subtype_dealloc will call tp_dealloc_func, which can call its + # base's tp_dealloc and we get recursion. So break the recursion by setting + # W_DateTime_Date->tp_dealloc + make_typedescr(W_DateTime_Date.typedef, + basestruct=PyDateTime_DateTime.TO, + attach=type_attach, + dealloc=date_dealloc, + ) + + make_typedescr(W_DateTime_Delta.typedef, + basestruct=PyDateTime_Delta.TO, + attach=timedeltatype_attach, + ) + +def type_attach(space, py_obj, w_obj, w_userdata=None): + '''Fills a newly allocated py_obj from the w_obj + ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return + py_datetime = rffi.cast(PyDateTime_Time, py_obj) + w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) + if space.is_none(w_tzinfo): + py_datetime.c_hastzinfo = cts.cast('unsigned char', 0) + py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO) + else: + py_datetime.c_hastzinfo = cts.cast('unsigned char', 1) + py_datetime.c_tzinfo = make_ref(space, w_tzinfo) + + at slot_function([PyObject], lltype.Void) +def type_dealloc(space, py_obj): + py_datetime = rffi.cast(PyDateTime_Time, py_obj) + if (widen(py_datetime.c_hastzinfo) != 0): + decref(space, py_datetime.c_tzinfo) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) + + at slot_function([PyObject], lltype.Void) +def date_dealloc(space, py_obj): + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) + +def timedeltatype_attach(space, py_obj, w_obj, w_userdata=None): + "Fills a newly allocated py_obj from the w_obj" + py_delta = rffi.cast(PyDateTime_Delta, py_obj) + days = space.int_w(space.getattr(w_obj, space.newtext('days'))) + py_delta.c_days = cts.cast('int', days) + seconds = space.int_w(space.getattr(w_obj, space.newtext('seconds'))) + py_delta.c_seconds = cts.cast('int', seconds) + microseconds = space.int_w(space.getattr(w_obj, space.newtext('microseconds'))) + py_delta.c_microseconds = cts.cast('int', microseconds) + # Constructors. They are better used as macros. @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, PyTypeObjectPtr], diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/include/datetime.h b/pypy/module/cpyext/include/datetime.h --- a/pypy/module/cpyext/include/datetime.h +++ b/pypy/module/cpyext/include/datetime.h @@ -4,49 +4,11 @@ extern "C" { #endif -/* Define structure for C API. */ -typedef struct { - /* type objects */ - PyTypeObject *DateType; - PyTypeObject *DateTimeType; - PyTypeObject *TimeType; - PyTypeObject *DeltaType; - PyTypeObject *TZInfoType; - - /* constructors */ - PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*); - PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, - PyObject*, PyTypeObject*); - PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); - PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*); -} PyDateTime_CAPI; +#include "cpyext_datetime.h" PyAPI_DATA(PyDateTime_CAPI*) PyDateTimeAPI; -#define PyDateTime_IMPORT \ - do { \ - if(PyDateTimeAPI==NULL) \ - PyDateTimeAPI = _PyDateTime_Import(); \ - } while (0) -typedef struct { - PyObject_HEAD -} PyDateTime_Delta; - -typedef struct { - PyObject_HEAD -} PyDateTime_Date; - -typedef struct { - PyObject_HEAD -} PyDateTime_Time; - -typedef struct { - PyObject_HEAD -} PyDateTime_DateTime; - -typedef struct { - PyObject_HEAD -} PyDateTime_TZInfo; +#define PyDateTime_IMPORT (PyDateTimeAPI = _PyDateTime_Import()) /* Macros for accessing constructors in a simplified fashion. */ #define PyDate_FromDate(year, month, day) \ diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h --- a/pypy/module/cpyext/include/longobject.h +++ b/pypy/module/cpyext/include/longobject.h @@ -18,6 +18,9 @@ PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS) #define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type) +#define _PyLong_AsByteArray(v, bytes, n, little_endian, is_signed) \ + _PyLong_AsByteArrayO((PyObject *)(v), bytes, n, little_endian, is_signed) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -3,9 +3,9 @@ cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t, CONST_STRING, ADDR, CANNOT_FAIL) from pypy.objspace.std.longobject import W_LongObject -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, oefmt from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask -from rpython.rlib.rbigint import rbigint +from rpython.rlib.rbigint import rbigint, InvalidSignednessError PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long") @@ -238,3 +238,26 @@ byteorder = 'big' result = rbigint.frombytes(s, byteorder, signed != 0) return space.newlong_from_rbigint(result) + + at cpython_api([PyObject, rffi.UCHARP, rffi.SIZE_T, + rffi.INT_real, rffi.INT_real], rffi.INT_real, error=-1) +def _PyLong_AsByteArrayO(space, w_v, bytes, n, little_endian, is_signed): + n = rffi.cast(lltype.Signed, n) + little_endian = rffi.cast(lltype.Signed, little_endian) + signed = rffi.cast(lltype.Signed, is_signed) != 0 + byteorder = 'little' if little_endian else 'big' + bigint = space.bigint_w(w_v) + try: + digits = bigint.tobytes(n, byteorder, signed) + except InvalidSignednessError: # < 0 but not 'signed' + # in this case, CPython raises OverflowError even though the C + # comments say it should raise TypeError + raise oefmt(space.w_OverflowError, + "can't convert negative long to unsigned") + except OverflowError: + raise oefmt(space.w_OverflowError, + "long too big to convert") + assert len(digits) == n + for i in range(n): + bytes[i] = rffi.cast(rffi.UCHAR, digits[i]) + return 0 diff --git a/pypy/module/cpyext/parse/cpyext_datetime.h b/pypy/module/cpyext/parse/cpyext_datetime.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/parse/cpyext_datetime.h @@ -0,0 +1,52 @@ +/* Define structure for C API. */ +typedef struct { + /* type objects */ + PyTypeObject *DateType; + PyTypeObject *DateTimeType; + PyTypeObject *TimeType; + PyTypeObject *DeltaType; + PyTypeObject *TZInfoType; + + /* constructors */ + PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*); + PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, + PyObject*, PyTypeObject*); + PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); + PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*); +} PyDateTime_CAPI; + +typedef struct +{ + PyObject_HEAD + int days; /* -MAX_DELTA_DAYS <= days <= MAX_DELTA_DAYS */ + int seconds; /* 0 <= seconds < 24*3600 is invariant */ + int microseconds; /* 0 <= microseconds < 1000000 is invariant */ +} PyDateTime_Delta; + +/* The datetime and time types have an optional tzinfo member, + * PyNone if hastzinfo is false. + */ +typedef struct +{ + PyObject_HEAD + unsigned char hastzinfo; + PyObject *tzinfo; +} PyDateTime_Time; + +typedef struct +{ + PyObject_HEAD + unsigned char hastzinfo; + PyObject *tzinfo; +} PyDateTime_DateTime; + + +typedef struct { + PyObject_HEAD +} PyDateTime_Date; + + +typedef struct { + PyObject_HEAD +} PyDateTime_TZInfo; + diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -10,6 +10,8 @@ PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject +from pypy.objspace.std.noneobject import W_NoneObject From pypy.commits at gmail.com Wed Feb 28 10:33:56 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Feb 2018 07:33:56 -0800 (PST) Subject: [pypy-commit] pypy default: declare array.array to be a read-write buffer. Fixes issue #2751 Message-ID: <5a96cbe4.02ba1c0a.b90e1.b324@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93906:7c772420134d Date: 2018-02-28 16:30 +0100 http://bitbucket.org/pypy/pypy/changeset/7c772420134d/ Log: declare array.array to be a read-write buffer. Fixes issue #2751 diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -711,7 +711,7 @@ return space.newtext(s) W_ArrayBase.typedef = TypeDef( - 'array.array', + 'array.array', None, None, "read-write", __new__ = interp2app(w_array), __len__ = interp2app(W_ArrayBase.descr_len), diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py --- a/pypy/module/cpyext/test/test_abstract.py +++ b/pypy/module/cpyext/test/test_abstract.py @@ -96,6 +96,16 @@ assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm) assert s == buffer_support.charbuffer_as_string(ro_mm) + def test_array(self): + import array + buffer_support = self.get_buffer_support() + + s = 'a\0x' + a = array.array('B', [5, 0, 10]) + + buffer_support.zero_out_writebuffer(a) + assert list(a) == [0, 0, 0] + def test_nonbuffer(self): # e.g. int buffer_support = self.get_buffer_support() From pypy.commits at gmail.com Wed Feb 28 13:15:35 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Feb 2018 10:15:35 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default Message-ID: <5a96f1c7.4dd91c0a.dabc3.65c5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r93907:5289cd302dff Date: 2018-02-28 16:53 +0100 http://bitbucket.org/pypy/pypy/changeset/5289cd302dff/ Log: merge default From pypy.commits at gmail.com Wed Feb 28 15:45:25 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Feb 2018 12:45:25 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Add myself to Leysin sprint Message-ID: <5a9714e5.0a86df0a.38280.65ba@mx.google.com> Author: Ronan Lamy Branch: extradoc Changeset: r5870:4f00d407dfa7 Date: 2018-02-28 20:45 +0000 http://bitbucket.org/pypy/extradoc/changeset/4f00d407dfa7/ Log: Add myself to Leysin sprint diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -19,6 +19,7 @@ Floris Bruynooghe 17.3/21.3 Ermina Antonio Cuni 18.3/23.3 Ermina, exact dates to be confirmed Alexander Schremmer 18.3/20.3 Ermina +Ronan Lamy 17.3/23.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there