From pypy.commits at gmail.com Sat Dec 1 12:47:27 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Dec 2018 09:47:27 -0800 (PST) Subject: [pypy-commit] pypy default: *Finally* managed to write a test from issue2904 Message-ID: <5c02c92f.1c69fb81.1896a.cd18@mx.google.com> Author: Armin Rigo Branch: Changeset: r95391:1f05288de5ac Date: 2018-12-01 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/1f05288de5ac/ Log: *Finally* managed to write a test from issue2904 diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -74,7 +74,7 @@ print "Loop:" print '\n'.join([str(o) for o in loop.operations]) print - if expected_short: + if expected_short or getattr(info, 'short_preamble', None): print "Short Preamble:" short = info.short_preamble print '\n'.join([str(o) for o in short]) @@ -9502,113 +9502,27 @@ def test_issue2904(self): py.test.skip("XXX issue 2904") ops = """ - [p0, p1, p2, p3, p4, i5, p6, i7, p8, p9, p10, p11, p12, p13] - debug_merge_point(0, 0, ' #36 LOAD_FAST') - guard_value(i5, 4) [] - guard_isnull(p3) [] - guard_nonnull(p11) [] - debug_merge_point(0, 0, ' #39 LOAD_CONST') - guard_value(p2, ConstPtr(myptr)) [] - debug_merge_point(0, 0, ' #42 COMPARE_OP') - guard_class(p11, ConstClass(intobj_immut_vtable)) [] + [p8, p10, p11] i17 = getfield_gc_i(p11, descr=immut_intval) i19 = int_gt(i17, 0) guard_false(i19) [] - debug_merge_point(0, 0, ' #45 POP_JUMP_IF_FALSE') - debug_merge_point(0, 0, ' #71 POP_BLOCK') - p20 = getfield_gc_r(p4, descr=nextdescr3) - i21 = getfield_gc_i(p4, descr=valuedescr3) - guard_value(i21, 4) [] - debug_merge_point(0, 0, ' #72 LOAD_FAST') - guard_nonnull(p10) [] - debug_merge_point(0, 0, ' #75 LOAD_CONST') - debug_merge_point(0, 0, ' #78 INPLACE_ADD') - guard_class(p10, ConstClass(intobj_immut_vtable)) [] i24 = getfield_gc_i(p10, descr=immut_intval) - i26 = int_add_ovf(i24, 1) - guard_no_overflow() [] + i26 = int_add(i24, 1) p27 = new_with_vtable(descr=immut_descr) setfield_gc(p27, i26, descr=immut_intval) - debug_merge_point(0, 0, ' #79 STORE_FAST') - debug_merge_point(0, 0, ' #82 JUMP_ABSOLUTE') - quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr) - guard_not_invalidated() [] - i30 = getfield_raw_i(140031323711392, descr=adescr) - i32 = int_lt(i30, 0) - guard_false(i32) [] - debug_merge_point(0, 0, ' #15 LOAD_FAST') - p34 = same_as_r(ConstPtr(tupleaddr)) - p36 = same_as_r(ConstPtr(nullptr)) - i38 = same_as_i(4) - i40 = same_as_i(15) - p42 = same_as_r(ConstPtr(nullptr)) - p44 = same_as_r(ConstPtr(nullptr)) - guard_future_condition() [] - guard_value(i38, 4) [] - guard_isnull(p36) [] - guard_nonnull(p27) [] - debug_merge_point(0, 0, ' #18 LOAD_CONST') - guard_value(p34, ConstPtr(tupleaddr)) [] - debug_merge_point(0, 0, ' #21 COMPARE_OP') - guard_class(p27, ConstClass(intobj_immut_vtable)) [] - i48 = getfield_gc_i(p27, descr=immut_intval) - i50 = int_lt(i48, 60000) - guard_true(i50) [] - debug_merge_point(0, 0, ' #24 POP_JUMP_IF_FALSE') - debug_merge_point(0, 0, ' #27 LOAD_FAST') - guard_nonnull(p8) [] - debug_merge_point(0, 0, ' #30 STORE_FAST') - debug_merge_point(0, 0, ' #33 SETUP_LOOP') - p51 = new_with_vtable(descr=nodesize) - setfield_gc(p51, 72, descr=valuedescr3) - setfield_gc(p51, 4, descr=immut_intval) - setfield_gc(p51, p20, descr=nextdescr3) - debug_merge_point(0, 0, ' #36 LOAD_FAST') - debug_merge_point(0, 0, ' #39 LOAD_CONST') - debug_merge_point(0, 0, ' #42 COMPARE_OP') - guard_class(p8, ConstClass(intobj_immut_vtable)) [] + i50 = int_ge(i26, 60000) + guard_false(i50) [] i55 = getfield_gc_i(p8, descr=immut_intval) i57 = int_gt(i55, 0) guard_true(i57) [] - debug_merge_point(0, 0, ' #45 POP_JUMP_IF_FALSE') - debug_merge_point(0, 0, ' #48 LOAD_FAST') - debug_merge_point(0, 0, ' #51 LOAD_CONST') - debug_merge_point(0, 0, ' #54 INPLACE_SUBTRACT') - i59 = int_sub_ovf(i55, 1) - guard_no_overflow() [] + i59 = int_sub(i55, 1) p60 = new_with_vtable(descr=immut_descr) setfield_gc(p60, i59, descr=immut_intval) - debug_merge_point(0, 0, ' #55 STORE_FAST') - debug_merge_point(0, 0, ' #58 LOAD_FAST') - guard_nonnull(p9) [] - debug_merge_point(0, 0, ' #61 LOAD_CONST') - debug_merge_point(0, 0, ' #64 INPLACE_ADD') - guard_class(p9, ConstClass(intobj_immut_vtable)) [] - i62 = getfield_gc_i(p9, descr=immut_intval) - i64 = int_add_ovf(i62, 1) - guard_no_overflow() [] - p65 = new_with_vtable(descr=immut_descr) - setfield_gc(p65, i64, descr=immut_intval) - debug_merge_point(0, 0, ' #65 STORE_FAST') - debug_merge_point(0, 0, ' #68 JUMP_ABSOLUTE') - quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr) - guard_not_invalidated() [] - i68 = getfield_raw_i(140031323711392, descr=adescr) - i70 = int_lt(i68, 0) - guard_false(i70) [] - debug_merge_point(0, 0, ' #36 LOAD_FAST') - p72 = same_as_r(ConstPtr(myptr)) - p74 = same_as_r(ConstPtr(nullptr)) - i76 = same_as_i(4) - i78 = same_as_i(36) - p80 = same_as_r(ConstPtr(nullptr)) - p82 = same_as_r(ConstPtr(nullptr)) - guard_future_condition() [] - jump(p0, p1, p72, p74, p51, i76, p6, i78, p8, p65, p27, p60, p80, p82, descr=) + jump(p8, p27, p60, descr=) """ # expected = a loop that does NOT end up passing the constant 0 in the final jump() self.optimize_loop(ops, ops, - jump_values=[None] * 14) + jump_values=[None] * 3) class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4732,3 +4732,36 @@ res = self.meta_interp(f, [2, 200]) assert res == f(2, 200) + + def test_issue2904(self): + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + set_param(driver, 'threshold', 4) + set_param(driver, 'trace_eagerness', 1) + height = IntVal(height) + c = 0 + h = height + total = 0 + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h.intval > 0: + h = IntVal(h.intval - 1) + total = total + 1 + else: + c = c + 1 + if c >= iterations: + return total + assert height.intval > 0 + h = IntVal(height.intval - 1) + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) From pypy.commits at gmail.com Sat Dec 1 13:07:25 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Dec 2018 10:07:25 -0800 (PST) Subject: [pypy-commit] pypy default: Get rid of that assert in the test Message-ID: <5c02cddd.1c69fb81.3027f.ad13@mx.google.com> Author: Armin Rigo Branch: Changeset: r95392:a41bd76d7c6b Date: 2018-12-01 20:06 +0200 http://bitbucket.org/pypy/pypy/changeset/a41bd76d7c6b/ Log: Get rid of that assert in the test diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4753,14 +4753,13 @@ while True: driver.jit_merge_point(iterations=iterations, total=total, c=c, height=height, h=h) - if h.intval > 0: + if h.intval != 0: h = IntVal(h.intval - 1) total = total + 1 else: c = c + 1 if c >= iterations: return total - assert height.intval > 0 h = IntVal(height.intval - 1) res = self.meta_interp(f, [2, 200]) From pypy.commits at gmail.com Sat Dec 1 13:12:25 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Dec 2018 10:12:25 -0800 (PST) Subject: [pypy-commit] pypy default: Ok, that fails even with no classes and virtuals at all Message-ID: <5c02cf09.1c69fb81.c23c3.6d9e@mx.google.com> Author: Armin Rigo Branch: Changeset: r95393:45ea82eb8031 Date: 2018-12-01 20:11 +0200 http://bitbucket.org/pypy/pypy/changeset/45ea82eb8031/ Log: Ok, that fails even with no classes and virtuals at all diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4737,15 +4737,9 @@ driver = JitDriver(greens = [], reds=['iterations', 'total', 'c', 'height', 'h']) - class IntVal: - _immutable_fields_ = ['intval'] - def __init__(self, value): - self.intval = value - def f(height, iterations): set_param(driver, 'threshold', 4) set_param(driver, 'trace_eagerness', 1) - height = IntVal(height) c = 0 h = height total = 0 @@ -4753,14 +4747,14 @@ while True: driver.jit_merge_point(iterations=iterations, total=total, c=c, height=height, h=h) - if h.intval != 0: - h = IntVal(h.intval - 1) + if h != 0: + h = h - 1 total = total + 1 else: c = c + 1 if c >= iterations: return total - h = IntVal(height.intval - 1) + h = height - 1 res = self.meta_interp(f, [2, 200]) assert res == f(2, 200) From pypy.commits at gmail.com Sat Dec 1 13:56:43 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 01 Dec 2018 10:56:43 -0800 (PST) Subject: [pypy-commit] pypy default: Very minimal reduction of the test for issue 2904. Message-ID: <5c02d96b.1c69fb81.c6c95.1f9e@mx.google.com> Author: Armin Rigo Branch: Changeset: r95394:cca908f95ba1 Date: 2018-12-01 20:56 +0200 http://bitbucket.org/pypy/pypy/changeset/cca908f95ba1/ Log: Very minimal reduction of the test for issue 2904. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9500,29 +9500,17 @@ self.optimize_loop(ops, expected) def test_issue2904(self): - py.test.skip("XXX issue 2904") - ops = """ - [p8, p10, p11] - i17 = getfield_gc_i(p11, descr=immut_intval) - i19 = int_gt(i17, 0) - guard_false(i19) [] - i24 = getfield_gc_i(p10, descr=immut_intval) - i26 = int_add(i24, 1) - p27 = new_with_vtable(descr=immut_descr) - setfield_gc(p27, i26, descr=immut_intval) - i50 = int_ge(i26, 60000) - guard_false(i50) [] - i55 = getfield_gc_i(p8, descr=immut_intval) - i57 = int_gt(i55, 0) - guard_true(i57) [] - i59 = int_sub(i55, 1) - p60 = new_with_vtable(descr=immut_descr) - setfield_gc(p60, i59, descr=immut_intval) - jump(p8, p27, p60, descr=) - """ - # expected = a loop that does NOT end up passing the constant 0 in the final jump() - self.optimize_loop(ops, ops, - jump_values=[None] * 3) + # we don't store advanced virtualstate information like "i1 = i2 + 1", + # which means that the following loop, when unrolled, cannot be + # optimized based on the knowledge that "i1 = i2 + 1" from the + # preamble---we can't use that knowledge. + ops = """ + [i1, i2] + guard_value(i1, 10) [] + i3 = int_add(i2, 1) + jump(i3, i2) + """ + self.optimize_loop(ops, ops) class TestLLtype(OptimizeOptTest, LLtypeMixin): pass From pypy.commits at gmail.com Sun Dec 2 07:38:57 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 02 Dec 2018 04:38:57 -0800 (PST) Subject: [pypy-commit] pypy default: ouch: scoped_alloc can itself allocate a GC object, the with handler! thus it Message-ID: <5c03d261.1c69fb81.79f52.06e4@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95395:74c1f4c8363d Date: 2018-12-02 13:38 +0100 http://bitbucket.org/pypy/pypy/changeset/74c1f4c8363d/ Log: ouch: scoped_alloc can itself allocate a GC object, the with handler! thus it can't be used here, now that time.time is used from the GC. diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,7 +136,8 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: - with lltype.scoped_alloc(TIMEVAL) as t: + t = lltype.malloc(TIMEVAL, flavor='raw') + try: errcode = -1 if GETTIMEOFDAY_NO_TZ: errcode = c_gettimeofday(t) @@ -145,13 +146,18 @@ if rffi.cast(rffi.LONG, errcode) == 0: result = decode_timeval(t) + finally: + lltype.free(t, flavor='raw') if result != -1: return result else: # assume using ftime(3) - with lltype.scoped_alloc(TIMEB) as t: + t = lltype.malloc(TIMEB, flavor='raw') + try: c_ftime(t) result = (float(intmask(t.c_time)) + float(intmask(t.c_millitm)) * 0.001) + finally: + lltype.free(t, flavor='raw') return result return float(c_time(void)) From pypy.commits at gmail.com Sun Dec 2 08:24:06 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 02 Dec 2018 05:24:06 -0800 (PST) Subject: [pypy-commit] pypy default: comment explaining the problem Message-ID: <5c03dcf6.1c69fb81.86244.612a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95396:b787df64587c Date: 2018-12-02 14:23 +0100 http://bitbucket.org/pypy/pypy/changeset/b787df64587c/ Log: comment explaining the problem diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,6 +136,8 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: + # NB: can't use lltype.scoped_malloc, because that will allocate the + # with handler in the GC, but we want to use time.time from gc.collect! t = lltype.malloc(TIMEVAL, flavor='raw') try: errcode = -1 From pypy.commits at gmail.com Sun Dec 2 11:52:07 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:07 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix for passing in public_encoder_name (and refactor) Message-ID: <5c040db7.1c69fb81.91fc9.8030@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95397:6e92ce356ff6 Date: 2018-11-29 14:23 -0800 http://bitbucket.org/pypy/pypy/changeset/6e92ce356ff6/ Log: test, fix for passing in public_encoder_name (and refactor) diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1042,15 +1042,18 @@ def str_decode_utf_16(s, errors, final=True, errorhandler=None): - return str_decode_utf_16_helper(s, errors, final, errorhandler, "native")[:3] + return str_decode_utf_16_helper(s, errors, final, errorhandler, + "native")[:3] def str_decode_utf_16_be(s, errors, final=True, errorhandler=None): - return str_decode_utf_16_helper(s, errors, final, errorhandler, "big")[:3] + return str_decode_utf_16_helper(s, errors, final, errorhandler, "big", + 'utf16-be')[:3] def str_decode_utf_16_le(s, errors, final=True, errorhandler=None): - return str_decode_utf_16_helper(s, errors, final, errorhandler, "little")[:3] + return str_decode_utf_16_helper(s, errors, final, errorhandler, "little", + 'utf16-le')[:3] def str_decode_utf_16_helper(s, errors, final=True, errorhandler=None, diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -654,6 +654,8 @@ b'\x00\x00\xd8\xae') assert (u'\x80\ud800'.encode('utf8', 'surrogatepass') == b'\xc2\x80\xed\xa0\x80') + assert b'\xd8\x03\xdf\xff\xdc\x80\x00A'.decode('utf_16_be', + 'surrogatepass') == u'\U00010fff\udc80A' def test_badandgoodsurrogatepassexceptions(self): import codecs From pypy.commits at gmail.com Sun Dec 2 11:52:09 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:09 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add failing test for charmaps Message-ID: <5c040db9.1c69fb81.a4ae4.b8ab@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95398:82cbb3e25737 Date: 2018-11-29 16:59 -0800 http://bitbucket.org/pypy/pypy/changeset/82cbb3e25737/ Log: add failing test for charmaps diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -1052,3 +1052,11 @@ errors = [] assert sin.encode("iso-8859-15", "test.record") == b"\xac\xa4" assert errors == [u'\u1234\u1234', u'\u8000'] + + def test_unmapped(self): + # from stdlib tests, bad byte: \xa5 is unmapped in iso-8859-3 + assert (b"foo\xa5bar".decode("iso-8859-3", "surrogateescape") == + "foo\udca5bar") + assert ("foo\udca5bar".encode("iso-8859-3", "surrogateescape") == + b"foo\xa5bar") + From pypy.commits at gmail.com Sun Dec 2 11:52:11 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:11 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: disallow tuple input to newtext, and also refactor some unicode/utf8 recoding Message-ID: <5c040dbb.1c69fb81.de854.a84e@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95399:5a6b7f57a324 Date: 2018-11-29 22:08 -0800 http://bitbucket.org/pypy/pypy/changeset/5a6b7f57a324/ Log: disallow tuple input to newtext, and also refactor some unicode/utf8 recoding diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -115,7 +115,7 @@ return W_FString(substr, rawmode) else: v = unicodehelper.str_decode_utf8(substr, 'strict', True, None) - return space.newtext(v) + return space.newtext(*v) v = PyString_DecodeEscape(space, substr, 'strict', encoding) return space.newbytes(v) diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -771,7 +771,7 @@ self._check_closed(space) self._writeflush(space) limit = convert_size(space, w_limit) - return space.newtext(self._readline(space, limit)) + return space.newtext(*self._readline(space, limit)) def _readline(self, space, limit): # This is a separate function so that readline_w() can be jitted. diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1154,7 +1154,7 @@ raise oefmt(space.w_ValueError, "array contains a unicode character out of " "range(0x110000)") - return space.newtext(item) + return space.newtext(rutf8.unichr_as_utf8(ord(item)), 1) assert 0, "unreachable" # interface diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -459,8 +459,8 @@ _set_module_object(space, "timezone", space.newint(timezone)) _set_module_object(space, 'daylight', space.newint(daylight)) - tzname_w = [space.newtext(tzname[0].decode('latin-1')), - space.newtext(tzname[1].decode('latin-1'))] + tzname_w = [space.newtext(tzname[0]), + space.newtext(tzname[1])] _set_module_object(space, 'tzname', space.newtuple(tzname_w)) _set_module_object(space, 'altzone', space.newint(altzone)) diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py --- a/pypy/objspace/std/marshal_impl.py +++ b/pypy/objspace/std/marshal_impl.py @@ -371,9 +371,9 @@ m.atom_str(TYPE_STRING, x.co_code) _marshal_tuple(space, x.co_consts_w, m) _marshal_tuple(space, x.co_names_w, m) # list of w_unicodes - co_varnames_w = [space.newtext(_decode_utf8(space, s)) for s in x.co_varnames] - co_freevars_w = [space.newtext(_decode_utf8(space, s)) for s in x.co_freevars] - co_cellvars_w = [space.newtext(_decode_utf8(space, s)) for s in x.co_cellvars] + co_varnames_w = [space.newtext(*_decode_utf8(space, s)) for s in x.co_varnames] + co_freevars_w = [space.newtext(*_decode_utf8(space, s)) for s in x.co_freevars] + co_cellvars_w = [space.newtext(*_decode_utf8(space, s)) for s in x.co_cellvars] _marshal_tuple(space, co_varnames_w, m) # more lists, now of w_unicodes _marshal_tuple(space, co_freevars_w, m) _marshal_tuple(space, co_cellvars_w, m) @@ -451,7 +451,7 @@ @unmarshaller(TYPE_UNICODE) def unmarshal_unicode(space, u, tc): uc = _decode_utf8(space, u.get_str()) - return space.newtext(uc) + return space.newtext(*uc) @unmarshaller(TYPE_INTERNED) def unmarshal_interned(space, u, tc): @@ -464,7 +464,7 @@ else: lng = u.get_lng() s = u.get(lng) - w_u = u.space.newtext(s.decode('latin-1')) + w_u = u.space.newtext(s) if interned: w_u = u.space.new_interned_w_str(w_u) return w_u diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -380,16 +380,16 @@ def newbytearray(self, l): return W_BytearrayObject(l) + # XXX TODO - remove this and force all users to call with utf8 @specialize.argtype(1) - def newtext(self, s, lgt=-1): + def newtext(self, s, lgt=-1, unused=-1): + # the unused argument can be from something like + # newtext(*decode_utf8sp(space, code)) if isinstance(s, unicode): s, lgt = s.encode('utf8'), len(s) - elif isinstance(s, str) and lgt < 0: + assert isinstance(s, str) + if lgt < 0: lgt = rutf8.codepoints_in_utf8(s) - elif isinstance(s, tuple): - # result of decode_utf8 - s, lgt, codepoints = s - assert isinstance(s, str) return W_UnicodeObject(s, lgt) def newtext_or_none(self, s, lgt=-1): From pypy.commits at gmail.com Sun Dec 2 11:52:13 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:13 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix failing tests Message-ID: <5c040dbd.1c69fb81.ed96f.6e5f@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95400:58cac9f73bf3 Date: 2018-12-01 05:19 -0800 http://bitbucket.org/pypy/pypy/changeset/58cac9f73bf3/ Log: fix failing tests diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1732,13 +1732,17 @@ r, newindex, rettype = errorhandler(errors, "charmap", "character maps to ", s, startindex, index) - for cp2 in rutf8.Utf8StringIterator(r): - ch2 = mapping.get(cp2, '') - if not ch2: - errorhandler( - "strict", "charmap", "character maps to ", - s, startindex, index) - result.append(ch2) + if rettype == 'u': + for cp2 in rutf8.Utf8StringIterator(r): + ch2 = mapping.get(cp2, '') + if not ch2: + errorhandler( + "strict", "charmap", "character maps to ", + s, startindex, index) + result.append(ch2) + else: + for ch in r: + result.append(ch) if index != newindex: # Should be uncommon index = newindex pos = rutf8._pos_at_index(s, newindex) diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py --- a/pypy/module/_locale/interp_locale.py +++ b/pypy/module/_locale/interp_locale.py @@ -41,8 +41,9 @@ def charp2uni(space, s): "Convert a char* pointer to unicode according to the current locale" - # XXX use mbstowcs() - return space.newtext(rffi.charp2str(s)) + w_val = space.newbytes(rffi.charp2str(s)) + return space.call_function(space.w_unicode, w_val, space.newtext('utf-8'), + space.newtext('surrogateescape')) def localeconv(space): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -146,9 +147,11 @@ Return the value for the locale information associated with key.""" try: - return space.newtext(rlocale.nl_langinfo(key)) + w_val = space.newbytes(rlocale.nl_langinfo(key)) except ValueError: raise oefmt(space.w_ValueError, "unsupported langinfo constant") + return space.call_function(space.w_unicode, w_val, + space.newtext('utf-8'), space.newtext('surrogateescape')) #___________________________________________________________________ # HAVE_LIBINTL dependence From pypy.commits at gmail.com Sun Dec 2 11:52:15 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:15 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix for unicode.center Message-ID: <5c040dbf.1c69fb81.0826.4c34@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95401:d54afa255930 Date: 2018-12-02 08:27 -0800 http://bitbucket.org/pypy/pypy/changeset/d54afa255930/ Log: test, fix for unicode.center diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -262,6 +262,7 @@ assert u'abc'.center(5, u'*') == u'*abc*' # Python 2.4 assert u'abc'.center(5, '*') == u'*abc*' # Python 2.4 raises(TypeError, u'abc'.center, 4, u'cba') + assert 'x'.center(2, u'\U0010FFFF') == u'x\U0010FFFF' def test_title(self): assert "brown fox".title() == "Brown Fox" diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -868,15 +868,14 @@ @unwrap_spec(width=int, w_fillchar=WrappedDefault(u' ')) def descr_center(self, space, width, w_fillchar): value = self._utf8 - fillchar = self.convert_arg_to_w_unicode(space, w_fillchar)._utf8 - if len(fillchar) != 1: + fillchar = space.utf8_w(w_fillchar) + if space.len_w(w_fillchar) != 1: raise oefmt(space.w_TypeError, "center() argument 2 must be a single character") d = width - self._len() if d > 0: offset = d//2 + (d & width & 1) - fillchar = fillchar[0] centered = offset * fillchar + value + (d - offset) * fillchar else: centered = value From pypy.commits at gmail.com Sun Dec 2 11:52:16 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:16 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add failing test Message-ID: <5c040dc0.1c69fb81.94084.c8de@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95402:ce732b3c6929 Date: 2018-12-02 08:27 -0800 http://bitbucket.org/pypy/pypy/changeset/ce732b3c6929/ Log: add failing test diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -148,6 +148,40 @@ lgt = 12 assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', lgt) + def test_unexpected_end_of_data(self): + """ + Test that an 'unexpected end of data' error is raised when the string + ends after a start byte of a 2-, 3-, or 4-bytes sequence without having + enough continuation bytes. The incomplete sequence is replaced with a + single U+FFFD when errors='replace'. + E.g. in the sequence , F3 is the start byte of a 4-bytes + sequence, but it's followed by only 2 valid continuation bytes and the + last continuation bytes is missing. + Note: the continuation bytes must be all valid, if one of them is + invalid another error will be raised. + """ + sequences = [ + 'C2', 'DF', + 'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF', + 'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF', + 'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF', + 'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF', + 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF', + 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF' + ] + FFFD = '\ufffd' + for seq in sequences: + print(seq) + bseq = bytes(int(c, 16) for c in seq.split()) + exc = raises(UnicodeDecodeError, bseq.decode, 'utf-8') + assert 'unexpected end of data' in str(exc.value) + assert bseq.decode('utf-8', 'replace') == u'\ufffd' + assert ((b'aaaa' + bseq + b'bbbb').decode('utf-8', 'replace') == + u'aaaa\ufffdbbbb') + assert bseq.decode('utf-8', 'ignore') == '' + assert ((b'aaaa' + bseq + b'bbbb').decode('utf-8', 'ignore') == + u'aaaabbbb') + class AppTestPartialEvaluation: spaceconfig = dict(usemodules=['array',]) From pypy.commits at gmail.com Sun Dec 2 11:52:18 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:18 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix for '%c' % uchr Message-ID: <5c040dc2.1c69fb81.b3ae5.dd85@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95403:8d78ef00bde9 Date: 2018-12-02 08:38 -0800 http://bitbucket.org/pypy/pypy/changeset/8d78ef00bde9/ Log: test, fix for '%c' % uchr diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -499,7 +499,7 @@ else: if space.isinstance_w(w_value, space.w_unicode): ustr = space.utf8_w(w_value) - if len(ustr) == 1: + if space.len_w(w_value) == 1: self.std_wp(ustr) return raise oefmt(space.w_TypeError, "%c requires int or char") diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -997,6 +997,9 @@ assert type(s) is str assert s == '\u1234' + def test_formatting_uchr(self): + assert '%c' % '\U00021483' == '\U00021483' + def test_formatting_unicode__str__(self): class A: def __init__(self, num): From pypy.commits at gmail.com Sun Dec 2 11:52:20 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 02 Dec 2018 08:52:20 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: tweak descr_capitalize Message-ID: <5c040dc4.1c69fb81.e49ea.3fff@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95404:0d983e9b4100 Date: 2018-12-02 08:51 -0800 http://bitbucket.org/pypy/pypy/changeset/0d983e9b4100/ Log: tweak descr_capitalize diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -849,8 +849,7 @@ return W_UnicodeObject(self._utf8[byte_start:byte_stop], stop - start) def descr_capitalize(self, space): - value = self._utf8 - if len(value) == 0: + if self._len() == 0: return self._empty() builder = rutf8.Utf8StringBuilder(len(self._utf8)) From pypy.commits at gmail.com Mon Dec 3 03:40:05 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 03 Dec 2018 00:40:05 -0800 (PST) Subject: [pypy-commit] pypy default: fix weird fallout of 74c1f4c8363d Message-ID: <5c04ebe5.1c69fb81.fb087.c3f8@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95405:52c3040f147b Date: 2018-12-03 09:39 +0100 http://bitbucket.org/pypy/pypy/changeset/52c3040f147b/ Log: fix weird fallout of 74c1f4c8363d diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -179,7 +179,7 @@ t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total0 + total == 9 def test_loop(): l = [10, 12, 15, 1] From pypy.commits at gmail.com Mon Dec 3 12:02:35 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 03 Dec 2018 09:02:35 -0800 (PST) Subject: [pypy-commit] pypy py3.5: use cpython's argument name (should maybe be done more systematically?) Message-ID: <5c0561ab.1c69fb81.299ec.7d62@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r95406:ce637db93b36 Date: 2018-12-03 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/ce637db93b36/ Log: use cpython's argument name (should maybe be done more systematically?) diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -97,8 +97,8 @@ w_obj = space.call_function(w_inttype, w_obj) return w_obj - @unwrap_spec(nbytes=int, byteorder='text', signed=bool) - def descr_to_bytes(self, space, nbytes, byteorder, signed=False): + @unwrap_spec(length=int, byteorder='text', signed=bool) + def descr_to_bytes(self, space, length, byteorder, signed=False): """to_bytes(...) int.to_bytes(length, byteorder, *, signed=False) -> bytes @@ -121,7 +121,7 @@ """ bigint = space.bigint_w(self) try: - byte_string = bigint.tobytes(nbytes, byteorder=byteorder, + byte_string = bigint.tobytes(length, byteorder=byteorder, signed=signed) except InvalidEndiannessError: raise oefmt(space.w_ValueError, diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -378,6 +378,7 @@ assert (-8388608).to_bytes(3, 'little', signed=True) == b'\x00\x00\x80' raises(OverflowError, (-5).to_bytes, 1, 'big') raises(ValueError, (-5).to_bytes, 1, 'foo') + assert 65535 .to_bytes(length=2, byteorder='big') == b'\xff\xff' def test_negative_zero(self): x = eval("-self._long(0)") From pypy.commits at gmail.com Tue Dec 4 05:00:39 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 04 Dec 2018 02:00:39 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: don't call GC hook recursively Message-ID: <5c065047.1c69fb81.f60ab.4d57@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95407:af1299797766 Date: 2018-12-04 10:59 +0100 http://bitbucket.org/pypy/pypy/changeset/af1299797766/ Log: don't call GC hook recursively diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -116,12 +116,24 @@ self.descr_set_on_gc_collect(space, space.w_None) -class GcMinorHookAction(AsyncAction): +class NoRecursiveAction(AsyncAction): + depth = 0 + + def perform(self, ec, frame): + if self.depth == 0: + try: + self.depth += 1 + return self._do_perform(ec, frame) + finally: + self.depth -= 1 + + +class GcMinorHookAction(NoRecursiveAction): total_memory_used = 0 pinned_objects = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -144,7 +156,7 @@ self.pinned_objects = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcMinorStats( self.count, self.duration, @@ -156,12 +168,12 @@ self.space.call_function(self.w_callable, w_stats) -class GcCollectStepHookAction(AsyncAction): +class GcCollectStepHookAction(NoRecursiveAction): oldstate = 0 newstate = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -184,7 +196,7 @@ self.newstate = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStepStats( self.count, self.duration, @@ -197,7 +209,7 @@ self.space.call_function(self.w_callable, w_stats) -class GcCollectHookAction(AsyncAction): +class GcCollectHookAction(NoRecursiveAction): num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -206,7 +218,7 @@ rawmalloc_bytes_after = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -227,7 +239,7 @@ self.rawmalloc_bytes_after = NonConstant(r_uint(42)) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStats(self.count, self.num_major_collects, self.arenas_count_before, diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -180,3 +180,22 @@ assert gc.hooks.on_gc_minor is None assert gc.hooks.on_gc_collect_step is None assert gc.hooks.on_gc_collect is None + + def test_no_recursive(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + self.fire_gc_minor(1, 2, 3) # won't fire NOW + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + # the duration for the 2nd call is 41, because it also counts the 1 + # which was fired recursively + assert lst == [ + (1, 10, 20, 30), + (2, 41, 50, 60), + ] From pypy.commits at gmail.com Tue Dec 4 15:04:49 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 12:04:49 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_code_extra.py Message-ID: <5c06dde1.1c69fb81.86784.1595@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95408:6097b204a9a6 Date: 2018-11-30 04:55 +0000 http://bitbucket.org/pypy/pypy/changeset/6097b204a9a6/ Log: Move test_code_extra.py diff --git a/pypy/module/test_lib_pypy/test_code_extra.py b/extra_tests/test_code.py rename from pypy/module/test_lib_pypy/test_code_extra.py rename to extra_tests/test_code.py From pypy.commits at gmail.com Tue Dec 4 15:04:52 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 12:04:52 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Make test work on CPython Message-ID: <5c06dde4.1c69fb81.7e829.03d8@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95409:21f49a179752 Date: 2018-11-30 05:44 +0000 http://bitbucket.org/pypy/pypy/changeset/21f49a179752/ Log: Make test work on CPython diff --git a/extra_tests/test_code.py b/extra_tests/test_code.py --- a/extra_tests/test_code.py +++ b/extra_tests/test_code.py @@ -1,4 +1,3 @@ -import py import sys import cStringIO import code @@ -14,6 +13,7 @@ finally: sys.stdout = old_stdout - if '__pypy__' not in sys.builtin_module_names: - py.test.skip('pypy only test') - assert mystdout.getvalue() == "5\n" + if '__pypy__' in sys.builtin_module_names: + assert mystdout.getvalue() == "5\n" + else: + assert mystdout.getvalue() == "5" From pypy.commits at gmail.com Tue Dec 4 22:31:43 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 19:31:43 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Test only the host's datetime, stop mixing direct and interpreted app-level tests Message-ID: <5c07469f.1c69fb81.849b.403d@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95410:a42566040c69 Date: 2018-12-05 01:03 +0000 http://bitbucket.org/pypy/pypy/changeset/a42566040c69/ Log: Test only the host's datetime, stop mixing direct and interpreted app-level tests diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,13 +1,9 @@ """Additional tests for datetime.""" +import datetime -from __future__ import absolute_import -import py -from pypy.module.test_lib_pypy.support import import_lib_pypy - -class BaseTestDatetime: +class TestDatetime: def test_repr(self): - import datetime checks = ( (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"), (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"), @@ -20,7 +16,6 @@ assert repr(obj) == expected def test_repr_overridden(self): - import datetime class date_safe(datetime.date): pass @@ -45,7 +40,6 @@ assert repr(obj) == expected def test_attributes(self): - import datetime for x in [datetime.date.today(), datetime.time(), datetime.datetime.utcnow(), @@ -54,7 +48,6 @@ raises(AttributeError, 'x.abc = 1') def test_timedelta_init_long(self): - import datetime td = datetime.timedelta(microseconds=20000000000000000000) assert td.days == 231481481 assert td.seconds == 41600 @@ -63,7 +56,6 @@ assert td.seconds == 41600 def test_unpickle(self): - import datetime e = raises(TypeError, datetime.date, '123') assert e.value.args[0] == 'an integer is required' e = raises(TypeError, datetime.time, '123') @@ -81,10 +73,6 @@ def test_strptime(self): import time, sys - import datetime - if sys.version_info < (2, 6): - py.test.skip("needs the _strptime module") - string = '2004-12-01 13:02:47' format = '%Y-%m-%d %H:%M:%S' expected = datetime.datetime(*(time.strptime(string, format)[0:6])) @@ -92,7 +80,6 @@ assert expected == got def test_datetime_rounding(self): - import datetime b = 0.0000001 a = 0.9999994 @@ -106,9 +93,6 @@ assert datetime.datetime.utcfromtimestamp(a).second == 1 def test_more_datetime_rounding(self): - import datetime - # this test verified on top of CPython 2.7 (using a plain - # "import datetime" above) expected_results = { -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', @@ -133,7 +117,6 @@ assert repr(dt) == expected_results[t] def test_utcfromtimestamp(self): - import datetime """Confirm that utcfromtimestamp and fromtimestamp give consistent results. Based on danchr's test script in https://bugs.pypy.org/issue986 @@ -159,19 +142,16 @@ time.tzset() def test_utcfromtimestamp_microsecond(self): - import datetime dt = datetime.datetime.utcfromtimestamp(0) assert isinstance(dt.microsecond, int) def test_default_args(self): - import datetime raises(TypeError, datetime.datetime) raises(TypeError, datetime.datetime, 10) raises(TypeError, datetime.datetime, 10, 10) datetime.datetime(10, 10, 10) def test_check_arg_types(self): - import datetime import decimal class Number: def __init__(self, value): @@ -218,7 +198,6 @@ raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10, 10.) def test_utcnow_microsecond(self): - import datetime import copy dt = datetime.datetime.utcnow() @@ -227,14 +206,12 @@ copy.copy(dt) def test_radd(self): - import datetime class X(object): def __radd__(self, other): return "radd" assert datetime.date(10, 10, 10) + X() == "radd" def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(self): - import datetime class Foo(datetime.tzinfo): def utcoffset(self, dt): return datetime.timedelta(0.1) @@ -250,7 +227,6 @@ assert str(exc.value) == "can't compare offset-naive and offset-aware times" def test_future_types_newint(self): - import datetime try: from future.types.newint import newint except ImportError: @@ -309,14 +285,12 @@ assert td_div_newint_int == td_div_newint_newint def test_return_types(self): - import datetime td = datetime.timedelta(5) assert type(td.total_seconds()) is float class sub(datetime.timedelta): pass assert type(+sub()) is datetime.timedelta def test_subclass_date(self): - import datetime # replace() should return a subclass but not call __new__ or __init__. class MyDate(datetime.date): forbidden = False @@ -332,7 +306,6 @@ assert d2 == datetime.date(2016, 2, 5) def test_subclass_time(self): - import datetime # replace() should return a subclass but not call __new__ or __init__. class MyTime(datetime.time): forbidden = False @@ -348,7 +321,6 @@ assert d2 == datetime.time(5, 2, 3) def test_subclass_datetime(self): - import datetime # replace() should return a subclass but not call __new__ or __init__. class MyDatetime(datetime.datetime): forbidden = False @@ -362,15 +334,3 @@ d2 = d.replace(hour=7) assert type(d2) is MyDatetime assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3) - - -class TestDatetimeHost(BaseTestDatetime): - pass - - -class AppTestDatetimePyPy(BaseTestDatetime): - spaceconfig = dict(usemodules=['__pypy__', 'struct']) - def setup_class(cls): - space = cls.space - #cls.w___pypy__ = import_lib_pypy(space, '__pypy__') - cls.w_datetime = import_lib_pypy(space, 'datetime') From pypy.commits at gmail.com Tue Dec 4 22:31:45 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 19:31:45 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Use test functions instead of a test class Message-ID: <5c0746a1.1c69fb81.7f66.ded9@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95411:88fcba069236 Date: 2018-12-05 01:51 +0000 http://bitbucket.org/pypy/pypy/changeset/88fcba069236/ Log: Use test functions instead of a test class diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -2,335 +2,334 @@ import datetime -class TestDatetime: - def test_repr(self): - checks = ( - (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"), - (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"), - (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"), - (datetime.timedelta(1), "datetime.timedelta(1)"), - (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"), - (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"), - ) - for obj, expected in checks: - assert repr(obj) == expected +def test_repr(): + checks = ( + (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"), + (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"), + (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"), + (datetime.timedelta(1), "datetime.timedelta(1)"), + (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"), + (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"), + ) + for obj, expected in checks: + assert repr(obj) == expected - def test_repr_overridden(self): - class date_safe(datetime.date): - pass +def test_repr_overridden(): + class date_safe(datetime.date): + pass - class datetime_safe(datetime.datetime): - pass + class datetime_safe(datetime.datetime): + pass - class time_safe(datetime.time): - pass + class time_safe(datetime.time): + pass - class timedelta_safe(datetime.timedelta): - pass + class timedelta_safe(datetime.timedelta): + pass - checks = ( - (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"), - (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"), - (time_safe(12, 34, 56), "time_safe(12, 34, 56)"), - (timedelta_safe(1), "timedelta_safe(1)"), - (timedelta_safe(1, 2), "timedelta_safe(1, 2)"), - (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"), - ) - for obj, expected in checks: - assert repr(obj) == expected + checks = ( + (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"), + (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"), + (time_safe(12, 34, 56), "time_safe(12, 34, 56)"), + (timedelta_safe(1), "timedelta_safe(1)"), + (timedelta_safe(1, 2), "timedelta_safe(1, 2)"), + (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"), + ) + for obj, expected in checks: + assert repr(obj) == expected - def test_attributes(self): - for x in [datetime.date.today(), - datetime.time(), - datetime.datetime.utcnow(), - datetime.timedelta(), - datetime.tzinfo()]: - raises(AttributeError, 'x.abc = 1') +def test_attributes(): + for x in [datetime.date.today(), + datetime.time(), + datetime.datetime.utcnow(), + datetime.timedelta(), + datetime.tzinfo()]: + raises(AttributeError, 'x.abc = 1') - def test_timedelta_init_long(self): - td = datetime.timedelta(microseconds=20000000000000000000) - assert td.days == 231481481 - assert td.seconds == 41600 - td = datetime.timedelta(microseconds=20000000000000000000.) - assert td.days == 231481481 - assert td.seconds == 41600 +def test_timedelta_init_long(): + td = datetime.timedelta(microseconds=20000000000000000000) + assert td.days == 231481481 + assert td.seconds == 41600 + td = datetime.timedelta(microseconds=20000000000000000000.) + assert td.days == 231481481 + assert td.seconds == 41600 - def test_unpickle(self): - e = raises(TypeError, datetime.date, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.time, '123') - assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.datetime, '123') - assert e.value.args[0] == 'an integer is required' +def test_unpickle(): + e = raises(TypeError, datetime.date, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.time, '123') + assert e.value.args[0] == 'an integer is required' + e = raises(TypeError, datetime.datetime, '123') + assert e.value.args[0] == 'an integer is required' - datetime.time('\x01' * 6, None) - exc = raises(TypeError, datetime.time, '\x01' * 6, 123) - assert str(exc.value) == "bad tzinfo state arg" + datetime.time('\x01' * 6, None) + exc = raises(TypeError, datetime.time, '\x01' * 6, 123) + assert str(exc.value) == "bad tzinfo state arg" - datetime.datetime('\x01' * 10, None) - exc = raises(TypeError, datetime.datetime, '\x01' * 10, 123) - assert str(exc.value) == "bad tzinfo state arg" + datetime.datetime('\x01' * 10, None) + exc = raises(TypeError, datetime.datetime, '\x01' * 10, 123) + assert str(exc.value) == "bad tzinfo state arg" - def test_strptime(self): - import time, sys - string = '2004-12-01 13:02:47' - format = '%Y-%m-%d %H:%M:%S' - expected = datetime.datetime(*(time.strptime(string, format)[0:6])) - got = datetime.datetime.strptime(string, format) - assert expected == got +def test_strptime(): + import time, sys + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got - def test_datetime_rounding(self): - b = 0.0000001 - a = 0.9999994 +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 - assert datetime.datetime.utcfromtimestamp(a).second == 1 + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 - def test_more_datetime_rounding(self): - expected_results = { - -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', - -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', - -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', - -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', - } - for t in sorted(expected_results): - dt = datetime.datetime.utcfromtimestamp(t) - assert repr(dt) == expected_results[t] +def test_more_datetime_rounding(): + expected_results = { + -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', + -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', + } + for t in sorted(expected_results): + dt = datetime.datetime.utcfromtimestamp(t) + assert repr(dt) == expected_results[t] - def test_utcfromtimestamp(self): - """Confirm that utcfromtimestamp and fromtimestamp give consistent results. +def test_utcfromtimestamp(): + """Confirm that utcfromtimestamp and fromtimestamp give consistent results. - Based on danchr's test script in https://bugs.pypy.org/issue986 - """ - import os - import time - if os.name == 'nt': - skip("setting os.environ['TZ'] ineffective on windows") - try: - prev_tz = os.environ.get("TZ") - os.environ["TZ"] = "GMT" - time.tzset() - for unused in xrange(100): - now = time.time() - delta = (datetime.datetime.utcfromtimestamp(now) - - datetime.datetime.fromtimestamp(now)) - assert delta.days * 86400 + delta.seconds == 0 - finally: - if prev_tz is None: - del os.environ["TZ"] - else: - os.environ["TZ"] = prev_tz - time.tzset() + Based on danchr's test script in https://bugs.pypy.org/issue986 + """ + import os + import time + if os.name == 'nt': + skip("setting os.environ['TZ'] ineffective on windows") + try: + prev_tz = os.environ.get("TZ") + os.environ["TZ"] = "GMT" + time.tzset() + for unused in xrange(100): + now = time.time() + delta = (datetime.datetime.utcfromtimestamp(now) - + datetime.datetime.fromtimestamp(now)) + assert delta.days * 86400 + delta.seconds == 0 + finally: + if prev_tz is None: + del os.environ["TZ"] + else: + os.environ["TZ"] = prev_tz + time.tzset() - def test_utcfromtimestamp_microsecond(self): - dt = datetime.datetime.utcfromtimestamp(0) - assert isinstance(dt.microsecond, int) +def test_utcfromtimestamp_microsecond(): + dt = datetime.datetime.utcfromtimestamp(0) + assert isinstance(dt.microsecond, int) - def test_default_args(self): - raises(TypeError, datetime.datetime) - raises(TypeError, datetime.datetime, 10) - raises(TypeError, datetime.datetime, 10, 10) - datetime.datetime(10, 10, 10) +def test_default_args(): + raises(TypeError, datetime.datetime) + raises(TypeError, datetime.datetime, 10) + raises(TypeError, datetime.datetime, 10, 10) + datetime.datetime(10, 10, 10) - def test_check_arg_types(self): - import decimal - class Number: - def __init__(self, value): - self.value = value - def __int__(self): - return self.value - class SubInt(int): pass - class SubLong(long): pass +def test_check_arg_types(): + import decimal + class Number: + def __init__(self, value): + self.value = value + def __int__(self): + return self.value + class SubInt(int): pass + class SubLong(long): pass - dt10 = datetime.datetime(10, 10, 10, 10, 10, 10, 10) - for xx in [10L, - decimal.Decimal(10), - decimal.Decimal('10.9'), - Number(10), - Number(10L), - SubInt(10), - SubLong(10), - Number(SubInt(10)), - Number(SubLong(10))]: - dtxx = datetime.datetime(xx, xx, xx, xx, xx, xx, xx) - assert dt10 == dtxx - assert type(dtxx.month) is int - assert type(dtxx.second) is int + dt10 = datetime.datetime(10, 10, 10, 10, 10, 10, 10) + for xx in [10L, + decimal.Decimal(10), + decimal.Decimal('10.9'), + Number(10), + Number(10L), + SubInt(10), + SubLong(10), + Number(SubInt(10)), + Number(SubLong(10))]: + dtxx = datetime.datetime(xx, xx, xx, xx, xx, xx, xx) + assert dt10 == dtxx + assert type(dtxx.month) is int + assert type(dtxx.second) is int - exc = raises(TypeError, datetime.datetime,10, 10, '10') - assert str(exc.value) == 'an integer is required' + exc = raises(TypeError, datetime.datetime,10, 10, '10') + assert str(exc.value) == 'an integer is required' - f10 = Number(10.9) - exc = raises(TypeError, datetime.datetime, 10, 10, f10) - assert str(exc.value) == '__int__ method should return an integer' + f10 = Number(10.9) + exc = raises(TypeError, datetime.datetime, 10, 10, f10) + assert str(exc.value) == '__int__ method should return an integer' - class Float(float): - pass - s10 = Float(10.9) - exc = raises(TypeError, datetime.datetime, 10, 10, s10) - assert str(exc.value) == 'integer argument expected, got float' + class Float(float): + pass + s10 = Float(10.9) + exc = raises(TypeError, datetime.datetime, 10, 10, s10) + assert str(exc.value) == 'integer argument expected, got float' - raises(TypeError, datetime.datetime, 10., 10, 10) - raises(TypeError, datetime.datetime, 10, 10., 10) - raises(TypeError, datetime.datetime, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10, 10.) + raises(TypeError, datetime.datetime, 10., 10, 10) + raises(TypeError, datetime.datetime, 10, 10., 10) + raises(TypeError, datetime.datetime, 10, 10, 10.) + raises(TypeError, datetime.datetime, 10, 10, 10, 10.) + raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10.) + raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10.) + raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10, 10.) - def test_utcnow_microsecond(self): - import copy +def test_utcnow_microsecond(): + import copy - dt = datetime.datetime.utcnow() - assert type(dt.microsecond) is int + dt = datetime.datetime.utcnow() + assert type(dt.microsecond) is int - copy.copy(dt) + copy.copy(dt) - def test_radd(self): - class X(object): - def __radd__(self, other): - return "radd" - assert datetime.date(10, 10, 10) + X() == "radd" +def test_radd(): + class X(object): + def __radd__(self, other): + return "radd" + assert datetime.date(10, 10, 10) + X() == "radd" - def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(self): - class Foo(datetime.tzinfo): - def utcoffset(self, dt): - return datetime.timedelta(0.1) - naive = datetime.datetime(2014, 9, 22) - aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) - exc = raises(TypeError, naive.__eq__, aware) - assert str(exc.value) == "can't compare offset-naive and offset-aware datetimes" - exc = raises(TypeError, naive.__sub__, aware) - assert str(exc.value) == "can't subtract offset-naive and offset-aware datetimes" - naive = datetime.time(7, 32, 12) - aware = datetime.time(7, 32, 12, tzinfo=Foo()) - exc = raises(TypeError, naive.__eq__, aware) - assert str(exc.value) == "can't compare offset-naive and offset-aware times" +def test_raises_if_passed_naive_datetime_and_start_or_end_time_defined(): + class Foo(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(0.1) + naive = datetime.datetime(2014, 9, 22) + aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) + exc = raises(TypeError, naive.__eq__, aware) + assert str(exc.value) == "can't compare offset-naive and offset-aware datetimes" + exc = raises(TypeError, naive.__sub__, aware) + assert str(exc.value) == "can't subtract offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) + aware = datetime.time(7, 32, 12, tzinfo=Foo()) + exc = raises(TypeError, naive.__eq__, aware) + assert str(exc.value) == "can't compare offset-naive and offset-aware times" - def test_future_types_newint(self): - try: - from future.types.newint import newint - except ImportError: - skip('requires future') +def test_future_types_newint(): + try: + from future.types.newint import newint + except ImportError: + skip('requires future') - dt_from_ints = datetime.datetime(2015, 12, 31, 12, 34, 56) - dt_from_newints = datetime.datetime(newint(2015), newint(12), newint(31), newint(12), newint(34), newint(56)) - dt_from_mixed = datetime.datetime(2015, newint(12), 31, newint(12), 34, newint(56)) - assert dt_from_ints == dt_from_newints - assert dt_from_newints == dt_from_mixed - assert dt_from_mixed == dt_from_ints + dt_from_ints = datetime.datetime(2015, 12, 31, 12, 34, 56) + dt_from_newints = datetime.datetime(newint(2015), newint(12), newint(31), newint(12), newint(34), newint(56)) + dt_from_mixed = datetime.datetime(2015, newint(12), 31, newint(12), 34, newint(56)) + assert dt_from_ints == dt_from_newints + assert dt_from_newints == dt_from_mixed + assert dt_from_mixed == dt_from_ints - d_from_int = datetime.date.fromtimestamp(1431216000) - d_from_newint = datetime.date.fromtimestamp(newint(1431216000)) - assert d_from_int == d_from_newint + d_from_int = datetime.date.fromtimestamp(1431216000) + d_from_newint = datetime.date.fromtimestamp(newint(1431216000)) + assert d_from_int == d_from_newint - dt_from_int = datetime.datetime.fromtimestamp(1431216000) - dt_from_newint = datetime.datetime.fromtimestamp(newint(1431216000)) - assert dt_from_int == dt_from_newint + dt_from_int = datetime.datetime.fromtimestamp(1431216000) + dt_from_newint = datetime.datetime.fromtimestamp(newint(1431216000)) + assert dt_from_int == dt_from_newint - dtu_from_int = datetime.datetime.utcfromtimestamp(1431216000) - dtu_from_newint = datetime.datetime.utcfromtimestamp(newint(1431216000)) - assert dtu_from_int == dtu_from_newint + dtu_from_int = datetime.datetime.utcfromtimestamp(1431216000) + dtu_from_newint = datetime.datetime.utcfromtimestamp(newint(1431216000)) + assert dtu_from_int == dtu_from_newint - td_from_int = datetime.timedelta(16565) - tds_from_int = datetime.timedelta(seconds=1431216000) - td_from_newint = datetime.timedelta(newint(16565)) - tds_from_newint = datetime.timedelta(seconds=newint(1431216000)) - assert td_from_int == tds_from_int - assert td_from_int == td_from_newint - assert td_from_int == tds_from_newint - assert tds_from_int == td_from_newint - assert tds_from_int == tds_from_newint - assert td_from_newint == tds_from_newint + td_from_int = datetime.timedelta(16565) + tds_from_int = datetime.timedelta(seconds=1431216000) + td_from_newint = datetime.timedelta(newint(16565)) + tds_from_newint = datetime.timedelta(seconds=newint(1431216000)) + assert td_from_int == tds_from_int + assert td_from_int == td_from_newint + assert td_from_int == tds_from_newint + assert tds_from_int == td_from_newint + assert tds_from_int == tds_from_newint + assert td_from_newint == tds_from_newint - td_mul_int_int = td_from_int * 2 - td_mul_int_newint = td_from_int * newint(2) - td_mul_newint_int = td_from_newint * 2 - td_mul_newint_newint = td_from_newint * newint(2) - assert td_mul_int_int == td_mul_int_newint - assert td_mul_int_int == td_mul_newint_int - assert td_mul_int_int == td_mul_newint_newint - assert td_mul_int_newint == td_mul_newint_int - assert td_mul_int_newint == td_mul_newint_newint - assert td_mul_newint_int == td_mul_newint_newint + td_mul_int_int = td_from_int * 2 + td_mul_int_newint = td_from_int * newint(2) + td_mul_newint_int = td_from_newint * 2 + td_mul_newint_newint = td_from_newint * newint(2) + assert td_mul_int_int == td_mul_int_newint + assert td_mul_int_int == td_mul_newint_int + assert td_mul_int_int == td_mul_newint_newint + assert td_mul_int_newint == td_mul_newint_int + assert td_mul_int_newint == td_mul_newint_newint + assert td_mul_newint_int == td_mul_newint_newint - td_div_int_int = td_from_int / 3600 - td_div_int_newint = td_from_int / newint(3600) - td_div_newint_int = td_from_newint / 3600 - td_div_newint_newint = td_from_newint / newint(3600) - assert td_div_int_int == td_div_int_newint - assert td_div_int_int == td_div_newint_int - assert td_div_int_int == td_div_newint_newint - assert td_div_int_newint == td_div_newint_int - assert td_div_int_newint == td_div_newint_newint - assert td_div_newint_int == td_div_newint_newint + td_div_int_int = td_from_int / 3600 + td_div_int_newint = td_from_int / newint(3600) + td_div_newint_int = td_from_newint / 3600 + td_div_newint_newint = td_from_newint / newint(3600) + assert td_div_int_int == td_div_int_newint + assert td_div_int_int == td_div_newint_int + assert td_div_int_int == td_div_newint_newint + assert td_div_int_newint == td_div_newint_int + assert td_div_int_newint == td_div_newint_newint + assert td_div_newint_int == td_div_newint_newint - def test_return_types(self): - td = datetime.timedelta(5) - assert type(td.total_seconds()) is float - class sub(datetime.timedelta): pass - assert type(+sub()) is datetime.timedelta +def test_return_types(): + td = datetime.timedelta(5) + assert type(td.total_seconds()) is float + class sub(datetime.timedelta): pass + assert type(+sub()) is datetime.timedelta - def test_subclass_date(self): - # replace() should return a subclass but not call __new__ or __init__. - class MyDate(datetime.date): - forbidden = False - def __new__(cls): - if cls.forbidden: FAIL - return datetime.date.__new__(cls, 2016, 2, 3) - def __init__(self, *args): - if self.forbidden: FAIL - d = MyDate() - d.forbidden = True - d2 = d.replace(day=5) - assert type(d2) is MyDate - assert d2 == datetime.date(2016, 2, 5) +def test_subclass_date(): + # replace() should return a subclass but not call __new__ or __init__. + class MyDate(datetime.date): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.date.__new__(cls, 2016, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDate() + d.forbidden = True + d2 = d.replace(day=5) + assert type(d2) is MyDate + assert d2 == datetime.date(2016, 2, 5) - def test_subclass_time(self): - # replace() should return a subclass but not call __new__ or __init__. - class MyTime(datetime.time): - forbidden = False - def __new__(cls): - if cls.forbidden: FAIL - return datetime.time.__new__(cls, 1, 2, 3) - def __init__(self, *args): - if self.forbidden: FAIL - d = MyTime() - d.forbidden = True - d2 = d.replace(hour=5) - assert type(d2) is MyTime - assert d2 == datetime.time(5, 2, 3) +def test_subclass_time(): + # replace() should return a subclass but not call __new__ or __init__. + class MyTime(datetime.time): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.time.__new__(cls, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyTime() + d.forbidden = True + d2 = d.replace(hour=5) + assert type(d2) is MyTime + assert d2 == datetime.time(5, 2, 3) - def test_subclass_datetime(self): - # replace() should return a subclass but not call __new__ or __init__. - class MyDatetime(datetime.datetime): - forbidden = False - def __new__(cls): - if cls.forbidden: FAIL - return datetime.datetime.__new__(cls, 2016, 4, 5, 1, 2, 3) - def __init__(self, *args): - if self.forbidden: FAIL - d = MyDatetime() - d.forbidden = True - d2 = d.replace(hour=7) - assert type(d2) is MyDatetime - assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3) +def test_subclass_datetime(): + # replace() should return a subclass but not call __new__ or __init__. + class MyDatetime(datetime.datetime): + forbidden = False + def __new__(cls): + if cls.forbidden: FAIL + return datetime.datetime.__new__(cls, 2016, 4, 5, 1, 2, 3) + def __init__(self, *args): + if self.forbidden: FAIL + d = MyDatetime() + d.forbidden = True + d2 = d.replace(hour=7) + assert type(d2) is MyDatetime + assert d2 == datetime.datetime(2016, 4, 5, 7, 2, 3) From pypy.commits at gmail.com Tue Dec 4 22:31:47 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 19:31:47 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Modernise tests Message-ID: <5c0746a3.1c69fb81.ac23e.e06a@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95412:b55f735fa45e Date: 2018-12-05 03:22 +0000 http://bitbucket.org/pypy/pypy/changeset/b55f735fa45e/ Log: Modernise tests diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,50 +1,50 @@ """Additional tests for datetime.""" +import pytest + import datetime -def test_repr(): - checks = ( - (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"), - (datetime.datetime(2015, 6, 8, 12, 34, 56), "datetime.datetime(2015, 6, 8, 12, 34, 56)"), - (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"), - (datetime.timedelta(1), "datetime.timedelta(1)"), - (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"), - (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"), - ) - for obj, expected in checks: - assert repr(obj) == expected +class date_safe(datetime.date): + pass -def test_repr_overridden(): - class date_safe(datetime.date): - pass +class datetime_safe(datetime.datetime): + pass - class datetime_safe(datetime.datetime): - pass +class time_safe(datetime.time): + pass - class time_safe(datetime.time): - pass +class timedelta_safe(datetime.timedelta): + pass - class timedelta_safe(datetime.timedelta): - pass + at pytest.mark.parametrize("obj, expected", [ + (datetime.date(2015, 6, 8), "datetime.date(2015, 6, 8)"), + (datetime.datetime(2015, 6, 8, 12, 34, 56), + "datetime.datetime(2015, 6, 8, 12, 34, 56)"), + (datetime.time(12, 34, 56), "datetime.time(12, 34, 56)"), + (datetime.timedelta(1), "datetime.timedelta(1)"), + (datetime.timedelta(1, 2), "datetime.timedelta(1, 2)"), + (datetime.timedelta(1, 2, 3), "datetime.timedelta(1, 2, 3)"), + (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"), + (datetime_safe(2015, 6, 8, 12, 34, 56), + "datetime_safe(2015, 6, 8, 12, 34, 56)"), + (time_safe(12, 34, 56), "time_safe(12, 34, 56)"), + (timedelta_safe(1), "timedelta_safe(1)"), + (timedelta_safe(1, 2), "timedelta_safe(1, 2)"), + (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"), +]) +def test_repr(obj, expected): + assert repr(obj) == expected - checks = ( - (date_safe(2015, 6, 8), "date_safe(2015, 6, 8)"), - (datetime_safe(2015, 6, 8, 12, 34, 56), "datetime_safe(2015, 6, 8, 12, 34, 56)"), - (time_safe(12, 34, 56), "time_safe(12, 34, 56)"), - (timedelta_safe(1), "timedelta_safe(1)"), - (timedelta_safe(1, 2), "timedelta_safe(1, 2)"), - (timedelta_safe(1, 2, 3), "timedelta_safe(1, 2, 3)"), - ) - for obj, expected in checks: - assert repr(obj) == expected - -def test_attributes(): - for x in [datetime.date.today(), - datetime.time(), - datetime.datetime.utcnow(), - datetime.timedelta(), - datetime.tzinfo()]: - raises(AttributeError, 'x.abc = 1') + at pytest.mark.parametrize("obj", [ + datetime.date.today(), + datetime.time(), + datetime.datetime.utcnow(), + datetime.timedelta(), + datetime.tzinfo(), +]) +def test_attributes(obj): + with pytest.raises(AttributeError): + obj.abc = 1 def test_timedelta_init_long(): td = datetime.timedelta(microseconds=20000000000000000000) @@ -55,19 +55,24 @@ assert td.seconds == 41600 def test_unpickle(): - e = raises(TypeError, datetime.date, '123') + with pytest.raises(TypeError) as e: + datetime.date('123') assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.time, '123') + with pytest.raises(TypeError) as e: + datetime.time('123') assert e.value.args[0] == 'an integer is required' - e = raises(TypeError, datetime.datetime, '123') + with pytest.raises(TypeError) as e: + datetime.datetime('123') assert e.value.args[0] == 'an integer is required' datetime.time('\x01' * 6, None) - exc = raises(TypeError, datetime.time, '\x01' * 6, 123) + with pytest.raises(TypeError) as exc: + datetime.time('\x01' * 6, 123) assert str(exc.value) == "bad tzinfo state arg" datetime.datetime('\x01' * 10, None) - exc = raises(TypeError, datetime.datetime, '\x01' * 10, 123) + with pytest.raises(TypeError) as exc: + datetime.datetime('\x01' * 10, 123) assert str(exc.value) == "bad tzinfo state arg" def test_strptime(): @@ -145,9 +150,12 @@ assert isinstance(dt.microsecond, int) def test_default_args(): - raises(TypeError, datetime.datetime) - raises(TypeError, datetime.datetime, 10) - raises(TypeError, datetime.datetime, 10, 10) + with pytest.raises(TypeError): + datetime.datetime() + with pytest.raises(TypeError): + datetime.datetime(10) + with pytest.raises(TypeError): + datetime.datetime(10, 10) datetime.datetime(10, 10, 10) def test_check_arg_types(): @@ -175,26 +183,36 @@ assert type(dtxx.month) is int assert type(dtxx.second) is int - exc = raises(TypeError, datetime.datetime,10, 10, '10') + with pytest.raises(TypeError) as exc: + datetime.datetime(0, 10, '10') assert str(exc.value) == 'an integer is required' f10 = Number(10.9) - exc = raises(TypeError, datetime.datetime, 10, 10, f10) + with pytest.raises(TypeError) as exc: + datetime.datetime(10, 10, f10) assert str(exc.value) == '__int__ method should return an integer' class Float(float): pass s10 = Float(10.9) - exc = raises(TypeError, datetime.datetime, 10, 10, s10) + with pytest.raises(TypeError) as exc: + datetime.datetime(10, 10, s10) assert str(exc.value) == 'integer argument expected, got float' - raises(TypeError, datetime.datetime, 10., 10, 10) - raises(TypeError, datetime.datetime, 10, 10., 10) - raises(TypeError, datetime.datetime, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10.) - raises(TypeError, datetime.datetime, 10, 10, 10, 10, 10, 10, 10.) + with pytest.raises(TypeError): + datetime.datetime(10., 10, 10) + with pytest.raises(TypeError): + datetime.datetime(10, 10., 10) + with pytest.raises(TypeError): + datetime.datetime(10, 10, 10.) + with pytest.raises(TypeError): + datetime.datetime(10, 10, 10, 10.) + with pytest.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10.) + with pytest.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10.) + with pytest.raises(TypeError): + datetime.datetime(10, 10, 10, 10, 10, 10, 10.) def test_utcnow_microsecond(): import copy @@ -216,13 +234,17 @@ return datetime.timedelta(0.1) naive = datetime.datetime(2014, 9, 22) aware = datetime.datetime(2014, 9, 22, tzinfo=Foo()) - exc = raises(TypeError, naive.__eq__, aware) + with pytest.raises(TypeError) as exc: + naive.__eq__(aware) assert str(exc.value) == "can't compare offset-naive and offset-aware datetimes" - exc = raises(TypeError, naive.__sub__, aware) + with pytest.raises(TypeError) as exc: + naive.__sub__(aware) assert str(exc.value) == "can't subtract offset-naive and offset-aware datetimes" + naive = datetime.time(7, 32, 12) aware = datetime.time(7, 32, 12, tzinfo=Foo()) - exc = raises(TypeError, naive.__eq__, aware) + with pytest.raises(TypeError) as exc: + naive.__eq__(aware) assert str(exc.value) == "can't compare offset-naive and offset-aware times" def test_future_types_newint(): From pypy.commits at gmail.com Tue Dec 4 23:18:20 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 20:18:20 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Fix test to not require a random external library Message-ID: <5c07518c.1c69fb81.fb087.8978@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95413:8b124f6d8e41 Date: 2018-12-05 03:54 +0000 http://bitbucket.org/pypy/pypy/changeset/8b124f6d8e41/ Log: Fix test to not require a random external library diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -248,10 +248,10 @@ assert str(exc.value) == "can't compare offset-naive and offset-aware times" def test_future_types_newint(): - try: - from future.types.newint import newint - except ImportError: - skip('requires future') + # Issue 2193 + class newint(long): + def __int__(self): + return self dt_from_ints = datetime.datetime(2015, 12, 31, 12, 34, 56) dt_from_newints = datetime.datetime(newint(2015), newint(12), newint(31), newint(12), newint(34), newint(56)) From pypy.commits at gmail.com Tue Dec 4 23:18:22 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 20:18:22 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_datetime to extra_tests/ Message-ID: <5c07518e.1c69fb81.3a464.08bd@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95414:df0d67038889 Date: 2018-12-05 04:10 +0000 http://bitbucket.org/pypy/pypy/changeset/df0d67038889/ Log: Move test_datetime to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/extra_tests/test_datetime.py rename from pypy/module/test_lib_pypy/test_datetime.py rename to extra_tests/test_datetime.py From pypy.commits at gmail.com Tue Dec 4 23:18:24 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 20:18:24 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_cPickle Message-ID: <5c075190.1c69fb81.b5f32.293f@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95415:55ef81f0789c Date: 2018-12-05 04:17 +0000 http://bitbucket.org/pypy/pypy/changeset/55ef81f0789c/ Log: Move test_cPickle diff --git a/pypy/module/test_lib_pypy/test_cPickle.py b/extra_tests/test_cPickle.py rename from pypy/module/test_lib_pypy/test_cPickle.py rename to extra_tests/test_cPickle.py --- a/pypy/module/test_lib_pypy/test_cPickle.py +++ b/extra_tests/test_cPickle.py @@ -1,14 +1,14 @@ -from __future__ import absolute_import -import py - -from lib_pypy import cPickle +import pytest +import cPickle def test_stack_underflow(): - py.test.raises(cPickle.UnpicklingError, cPickle.loads, "a string") + with pytest.raises(cPickle.UnpicklingError): + cPickle.loads("a string") def test_bad_key(): - e = py.test.raises(cPickle.UnpicklingError, cPickle.loads, "v") - assert str(e.value) == "invalid load key, 'v'." + with pytest.raises(cPickle.UnpicklingError) as excinfo: + cPickle.loads("v") + assert str(excinfo.value) == "invalid load key, 'v'." def test_find_global(): import time, cStringIO @@ -23,7 +23,8 @@ f = cStringIO.StringIO(f.getvalue()) up = cPickle.Unpickler(f) up.find_global = None - e = py.test.raises(cPickle.UnpicklingError, up.load) + with pytest.raises(cPickle.UnpicklingError) as e: + up.load() assert str(e.value) == "Global and instance pickles are not supported." f = cStringIO.StringIO(f.getvalue()) From pypy.commits at gmail.com Wed Dec 5 00:00:25 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 21:00:25 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_functools Message-ID: <5c075b69.1c69fb81.73c1e.da42@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95416:57c63493d98d Date: 2018-12-05 04:55 +0000 http://bitbucket.org/pypy/pypy/changeset/57c63493d98d/ Log: Move test_functools diff --git a/pypy/module/test_lib_pypy/test_functools.py b/extra_tests/test_functools.py rename from pypy/module/test_lib_pypy/test_functools.py rename to extra_tests/test_functools.py --- a/pypy/module/test_lib_pypy/test_functools.py +++ b/extra_tests/test_functools.py @@ -1,6 +1,6 @@ import pytest -from lib_pypy import _functools +import _functools def test_partial_reduce(): From pypy.commits at gmail.com Wed Dec 5 00:00:26 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 21:00:26 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_gdbm_extra Message-ID: <5c075b6a.1c69fb81.5f9d1.74a0@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95417:22d96ea1b35e Date: 2018-12-05 04:59 +0000 http://bitbucket.org/pypy/pypy/changeset/22d96ea1b35e/ Log: Move test_gdbm_extra diff --git a/pypy/module/test_lib_pypy/test_gdbm_extra.py b/extra_tests/test_gdbm.py rename from pypy/module/test_lib_pypy/test_gdbm_extra.py rename to extra_tests/test_gdbm.py --- a/pypy/module/test_lib_pypy/test_gdbm_extra.py +++ b/extra_tests/test_gdbm.py @@ -1,13 +1,8 @@ -from __future__ import absolute_import -import py -from rpython.tool.udir import udir -try: - from lib_pypy import gdbm -except ImportError as e: - py.test.skip(e) +import pytest +gdbm = pytest.importorskip('gdbm') -def test_len(): - path = str(udir.join('test_gdbm_extra')) +def test_len(tmpdir): + path = str(tmpdir.join('test_gdbm_extra')) g = gdbm.open(path, 'c') g['abc'] = 'def' assert len(g) == 1 @@ -16,6 +11,6 @@ del g['abc'] assert len(g) == 1 -def test_unicode(): - path = unicode(udir.join('test_gdm_unicode')) +def test_unicode(tmpdir): + path = unicode(tmpdir.join('test_gdm_unicode')) g = gdbm.open(path, 'c') # does not crash From pypy.commits at gmail.com Wed Dec 5 01:32:38 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 22:32:38 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Convert AppTest to direct app-level test Message-ID: <5c077106.1c69fb81.ed96f.156f@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95418:c68624833582 Date: 2018-12-05 05:21 +0000 http://bitbucket.org/pypy/pypy/changeset/c68624833582/ Log: Convert AppTest to direct app-level test diff --git a/pypy/module/test_lib_pypy/test_cStringIO.py b/pypy/module/test_lib_pypy/test_cStringIO.py --- a/pypy/module/test_lib_pypy/test_cStringIO.py +++ b/pypy/module/test_lib_pypy/test_cStringIO.py @@ -1,25 +1,23 @@ """ Tests for the PyPy cStringIO implementation. """ +from cStringIO import StringIO -class AppTestcStringIO: - def setup_class(cls): - cls.w_io = cls.space.appexec([], "(): import cStringIO; return cStringIO") - cls.w_bytes = cls.space.wrap('some bytes') +data = b"some bytes" - def test_reset(self): - """ - Test that the reset method of cStringIO objects sets the position - marker to the beginning of the stream. - """ - io = self.io.StringIO() - io.write(self.bytes) - assert io.read() == '' - io.reset() - assert io.read() == self.bytes +def test_reset(): + """ + Test that the reset method of cStringIO objects sets the position + marker to the beginning of the stream. + """ + stream = StringIO() + stream.write(data) + assert stream.read() == '' + stream.reset() + assert stream.read() == data - io = self.io.StringIO(self.bytes) - assert io.read() == self.bytes - assert io.read() == '' - io.reset() - assert io.read() == self.bytes + stream = StringIO(data) + assert stream.read() == data + assert stream.read() == '' + stream.reset() + assert stream.read() == data From pypy.commits at gmail.com Wed Dec 5 01:32:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 22:32:40 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_cStringIO to extra_tests/ Message-ID: <5c077108.1c69fb81.6534c.4944@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95419:c6d22edc7732 Date: 2018-12-05 05:24 +0000 http://bitbucket.org/pypy/pypy/changeset/c6d22edc7732/ Log: Move test_cStringIO to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_cStringIO.py b/extra_tests/test_cStringIO.py rename from pypy/module/test_lib_pypy/test_cStringIO.py rename to extra_tests/test_cStringIO.py From pypy.commits at gmail.com Wed Dec 5 01:32:42 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Dec 2018 22:32:42 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Update and move test_dbm Message-ID: <5c07710a.1c69fb81.6534c.4947@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95420:74d03670644b Date: 2018-12-05 06:29 +0000 http://bitbucket.org/pypy/pypy/changeset/74d03670644b/ Log: Update and move test_dbm diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/extra_tests/test_dbm.py rename from pypy/module/test_lib_pypy/test_dbm_extra.py rename to extra_tests/test_dbm.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/extra_tests/test_dbm.py @@ -1,47 +1,53 @@ -from __future__ import absolute_import -import py, os -from rpython.tool.udir import udir -try: - from lib_pypy import dbm -except ImportError as e: - py.test.skip(e) +import pytest -import sys -if '__pypy__' not in sys.builtin_module_names: - skip("lib_pypy.dbm requires PyPy's ctypes") +import os -def test_get(): - path = str(udir.join('test_dbm_extra.test_get')) +dbm = pytest.importorskip('dbm') + + +def test_get(tmpdir): + path = str(tmpdir.join('test_dbm_extra.test_get')) d = dbm.open(path, 'c') x = d.get("42") assert x is None d.close() -def test_delitem(): - path = str(udir.join('test_dbm_extra.test_delitem')) +def test_delitem(tmpdir): + path = str(tmpdir.join('test_dbm_extra.test_delitem')) d = dbm.open(path, 'c') - py.test.raises(KeyError, "del d['xyz']") + with pytest.raises(KeyError): + del d['xyz'] -def test_nonstring(): - path = str(udir.join('test_dbm_extra.test_nonstring')) +def test_nonstring(tmpdir): + path = str(tmpdir.join('test_dbm_extra.test_nonstring')) d = dbm.open(path, 'c') - py.test.raises(TypeError, "d[123] = 'xyz'") - py.test.raises(TypeError, "d['xyz'] = 123") - py.test.raises(TypeError, "d['xyz'] = None") - py.test.raises(TypeError, "del d[123]") - py.test.raises(TypeError, "d[123]") - py.test.raises(TypeError, "123 in d") - py.test.raises(TypeError, "d.has_key(123)") - py.test.raises(TypeError, "d.setdefault(123, 'xyz')") - py.test.raises(TypeError, "d.setdefault('xyz', 123)") - py.test.raises(TypeError, "d.get(123)") + with pytest.raises(TypeError): + d[123] = 'xyz' + with pytest.raises(TypeError): + d['xyz'] = 123 + with pytest.raises(TypeError): + d['xyz'] = None + with pytest.raises(TypeError): + del d[123] + with pytest.raises(TypeError): + d[123] + with pytest.raises(TypeError): + 123 in d + with pytest.raises(TypeError): + d.has_key(123) + with pytest.raises(TypeError): + d.setdefault(123, 'xyz') + with pytest.raises(TypeError): + d.setdefault('xyz', 123) + with pytest.raises(TypeError): + d.get(123) assert dict(d) == {} d.setdefault('xyz', '123') assert dict(d) == {'xyz': '123'} d.close() -def test_multiple_sets(): - path = str(udir.join('test_dbm_extra.test_multiple_sets')) +def test_multiple_sets(tmpdir): + path = str(tmpdir.join('test_dbm_extra.test_multiple_sets')) d = dbm.open(path, 'c') d['xyz'] = '12' d['xyz'] = '3' @@ -49,9 +55,12 @@ assert dict(d) == {'xyz': '546'} assert d['xyz'] == '546' + at pytest.mark.skipif("'__pypy__' not in sys.modules") def test_extra(): - py.test.raises(TypeError, dbm.datum, 123) - py.test.raises(TypeError, dbm.datum, False) + with pytest.raises(TypeError): + dbm.datum(123) + with pytest.raises(TypeError): + dbm.datum(False) def test_null(): db = dbm.open('test', 'c') @@ -62,12 +71,12 @@ assert db['1'] == 'a\x00b' db.close() -def test_key_with_empty_value(): +def test_key_with_empty_value(tmpdir): # this test fails on CPython too (at least on tannit), and the # case shows up when gdbm is not installed and test_anydbm.py # falls back dbm. - py.test.skip("test may fail on CPython too") - path = str(udir.join('test_dbm_extra.test_key_with_empty_value')) + pytest.skip("test may fail on CPython too") + path = str(tmpdir.join('test_dbm_extra.test_key_with_empty_value')) d = dbm.open(path, 'c') assert 'key_with_empty_value' not in d d['key_with_empty_value'] = '' @@ -75,7 +84,7 @@ assert d['key_with_empty_value'] == '' d.close() -def test_unicode_filename(): - path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' +def test_unicode_filename(tmpdir): + path = str(tmpdir) + os.sep + u'test_dbm_extra.test_unicode_filename' d = dbm.open(path, 'c') d.close() From pypy.commits at gmail.com Wed Dec 5 03:57:46 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 05 Dec 2018 00:57:46 -0800 (PST) Subject: [pypy-commit] pypy default: Issue #2917 Message-ID: <5c07930a.1c69fb81.8e4ab.44d4@mx.google.com> Author: Armin Rigo Branch: Changeset: r95424:8f0f4e09382a Date: 2018-12-05 10:57 +0200 http://bitbucket.org/pypy/pypy/changeset/8f0f4e09382a/ Log: Issue #2917 Try to handle a bit better bogus PYTHONWARNINGS diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -182,6 +182,8 @@ module = category[:i] klass = category[i+1:] try: + if not module: + raise ImportError # instead of the ValueError we'd get m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -609,8 +609,14 @@ warnoptions.extend(pythonwarnings.split(',')) if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available From pypy.commits at gmail.com Wed Dec 5 11:34:01 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 05 Dec 2018 08:34:01 -0800 (PST) Subject: [pypy-commit] pypy default: Tentative fix for issue #2904 Message-ID: <5c07fdf9.1c69fb81.578db.c154@mx.google.com> Author: Armin Rigo Branch: Changeset: r95425:a318a9cf2b74 Date: 2018-12-05 18:32 +0200 http://bitbucket.org/pypy/pypy/changeset/a318a9cf2b74/ Log: Tentative fix for issue #2904 diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -11,7 +11,7 @@ from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - AbstractResOp, GuardResOp + GuardResOp from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -6,7 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rlib.rarithmetic import intmask @@ -44,8 +44,9 @@ if b.has_lower and b.has_upper and b.lower == b.upper: self.make_constant_int(box, b.lower) - if isinstance(box, AbstractResOp): - dispatch_bounds_ops(self, box) + box1 = self.optimizer.as_operation(box) + if box1 is not None: + dispatch_bounds_ops(self, box1) def _optimize_guard_true_false_value(self, op): return self.emit(op) @@ -126,7 +127,8 @@ v1, v2 = v2, v1 # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): - if not self.optimizer.is_inputarg(arg1): + arg1 = self.optimizer.as_operation(arg1) + if arg1 is not None and 0: # XXX check and re-enable this if arg1.getopnum() == rop.INT_ADD: prod_arg1 = arg1.getarg(0) prod_arg2 = arg1.getarg(1) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -288,15 +288,14 @@ self._really_emitted_operation = None self._last_guard_op = None + self._inparg_dict = {} self.set_optimizations(optimizations) self.setup() - def init_inparg_dict_from(self, lst): - pass - #self.inparg_dict = {} - #for box in lst: - # self.inparg_dict[box] = None + def add_to_inparg_dict_from(self, lst): + for box in lst: + self._inparg_dict[box] = None def set_optimizations(self, optimizations): if optimizations: @@ -385,9 +384,12 @@ return info.force_box(op, optforce) return op - def is_inputarg(self, op): - return True - #return op in self.inparg_dict + def as_operation(self, op): + # You should never check "isinstance(op, AbstractResOp" directly. + # Instead, use this helper. + if isinstance(op, AbstractResOp) and op not in self._inparg_dict: + return op + return None def get_constant_box(self, box): box = self.get_box_replacement(box) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers, AbstractResOp + OpHelpers from rpython.rlib.rarithmetic import highest_bit from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper import rclass @@ -490,11 +490,11 @@ def postprocess_GUARD_TRUE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_TRUE): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0) + self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0) self.make_constant(box, CONST_1) def optimize_GUARD_FALSE(self, op): @@ -502,11 +502,11 @@ def postprocess_GUARD_FALSE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_ZERO): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1) + self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1) self.make_constant(box, CONST_0) def optimize_ASSERT_NOT_NONE(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9503,14 +9503,21 @@ # we don't store advanced virtualstate information like "i1 = i2 + 1", # which means that the following loop, when unrolled, cannot be # optimized based on the knowledge that "i1 = i2 + 1" from the - # preamble---we can't use that knowledge. + # preamble---we can't use that knowledge. After the fix, we get + # the value "i2 + 1" passed as a third argument, possibly different + # from "i1". ops = """ [i1, i2] guard_value(i1, 10) [] i3 = int_add(i2, 1) jump(i3, i2) """ - self.optimize_loop(ops, ops) + expected = """ + [i1, i2, i3] + guard_value(i1, 10) [] + jump(i3, i2, i3) + """ + self.optimize_loop(ops, expected) class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -10,8 +10,7 @@ from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo from rpython.jit.metainterp.optimizeopt.virtualstate import ( VirtualStateConstructor, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints @@ -144,7 +143,7 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.init_inparg_dict_from(label_args) + self.optimizer.add_to_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) @@ -431,8 +430,9 @@ for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): - if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): - info = self.optimizer.getinfo(arg.getarg(0)) + arg1 = self.optimizer.as_operation(arg) + if arg1 is not None and rop.is_same_as(arg1.opnum): + info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -6,8 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.optimizeopt import info from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.unroll import unrolling_iterable From pypy.commits at gmail.com Wed Dec 5 13:33:49 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 05 Dec 2018 10:33:49 -0800 (PST) Subject: [pypy-commit] pypy default: Re-enable this intbounds optimization. Found that tracking resops that are Message-ID: <5c081a0d.1c69fb81.7509d.ac09@mx.google.com> Author: Armin Rigo Branch: Changeset: r95426:3c4bff197623 Date: 2018-12-05 20:33 +0200 http://bitbucket.org/pypy/pypy/changeset/3c4bff197623/ Log: Re-enable this intbounds optimization. Found that tracking resops that are *not* in the current loop is hard and error-prone; instead, track resops that *are* in the current loop, and it's OK if we occasionally miss some. diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -128,7 +128,7 @@ # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): arg1 = self.optimizer.as_operation(arg1) - if arg1 is not None and 0: # XXX check and re-enable this + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: prod_arg1 = arg1.getarg(0) prod_arg2 = arg1.getarg(1) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -279,6 +279,7 @@ self.quasi_immutable_deps = None self.replaces_guard = {} self._newoperations = [] + self._emittedoperations = {} self.optimizer = self self.optpure = None self.optheap = None @@ -288,15 +289,10 @@ self._really_emitted_operation = None self._last_guard_op = None - self._inparg_dict = {} self.set_optimizations(optimizations) self.setup() - def add_to_inparg_dict_from(self, lst): - for box in lst: - self._inparg_dict[box] = None - def set_optimizations(self, optimizations): if optimizations: self.first_optimization = optimizations[0] @@ -387,7 +383,7 @@ def as_operation(self, op): # You should never check "isinstance(op, AbstractResOp" directly. # Instead, use this helper. - if isinstance(op, AbstractResOp) and op not in self._inparg_dict: + if isinstance(op, AbstractResOp) and op in self._emittedoperations: return op return None @@ -407,6 +403,7 @@ def clear_newoperations(self): self._newoperations = [] + self._emittedoperations = {} def make_equal_to(self, op, newop): op = self.get_box_replacement(op) @@ -634,6 +631,7 @@ self._last_guard_op = None self._really_emitted_operation = op self._newoperations.append(op) + self._emittedoperations[op] = None def emit_guard_operation(self, op, pendingfields): guard_op = op # self.replace_op_with(op, op.getopnum()) @@ -678,6 +676,7 @@ return newop = self.replace_op_with_no_ovf(op) self._newoperations[-1] = newop + self._emittedoperations[newop] = None def replace_op_with_no_ovf(self, op): if op.getopnum() == rop.INT_MUL_OVF: @@ -722,6 +721,7 @@ new_descr = new_op.getdescr() new_descr.copy_all_attributes_from(old_descr) self._newoperations[old_op_pos] = new_op + self._emittedoperations[new_op] = None def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) @@ -9063,6 +9062,7 @@ self.optimize_loop(ops, expected) def test_same_as_preserves_info_in_the_preamble_2(self): + py.test.xfail("less efficient loop, investigate") ops = """ [i0, p0] ifoo = getfield_gc_i(p0, descr=valuedescr) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -21,7 +21,6 @@ if self.optunroll.short_preamble_producer is None: assert False # unreachable code op = preamble_op.op - #----self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, preamble_op.preamble_op, self) @@ -143,7 +142,6 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.add_to_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) From pypy.commits at gmail.com Wed Dec 5 14:11:00 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 05 Dec 2018 11:11:00 -0800 (PST) Subject: [pypy-commit] pypy default: It turns out re-enabling this optimization shows a failure in the metainterp Message-ID: <5c0822c4.1c69fb81.a188c.7690@mx.google.com> Author: Armin Rigo Branch: Changeset: r95427:d7e235d0e787 Date: 2018-12-05 20:42 +0200 http://bitbucket.org/pypy/pypy/changeset/d7e235d0e787/ Log: It turns out re-enabling this optimization shows a failure in the metainterp tests. Need to investigate (but later). diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -128,7 +128,7 @@ # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): arg1 = self.optimizer.as_operation(arg1) - if arg1 is not None: + if 0: # arg1 is not None: investigate, bug in test_loop_unroll.py if arg1.getopnum() == rop.INT_ADD: prod_arg1 = arg1.getarg(0) prod_arg2 = arg1.getarg(1) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,6 +4017,7 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) + ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i0, 2) + i29 = int_add(i28, 1) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i0, 2) + i29 = int_add(i28, 1) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,6 +6392,7 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) + i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) From pypy.commits at gmail.com Thu Dec 6 02:23:23 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 05 Dec 2018 23:23:23 -0800 (PST) Subject: [pypy-commit] pypy default: Redo d7e235d0e787 with the missing get_box_replacement() calls Message-ID: <5c08ce6b.1c69fb81.f5f71.c1ef@mx.google.com> Author: Armin Rigo Branch: Changeset: r95428:258a0be8f360 Date: 2018-12-06 09:05 +0200 http://bitbucket.org/pypy/pypy/changeset/258a0be8f360/ Log: Redo d7e235d0e787 with the missing get_box_replacement() calls diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -128,10 +128,10 @@ # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): arg1 = self.optimizer.as_operation(arg1) - if 0: # arg1 is not None: investigate, bug in test_loop_unroll.py + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = arg1.getarg(0) - prod_arg2 = arg1.getarg(1) + prod_arg1 = self.get_box_replacement(arg1.getarg(0)) + prod_arg2 = self.get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) From pypy.commits at gmail.com Thu Dec 6 11:59:58 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 06 Dec 2018 08:59:58 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Convert test_grp to regular (app-level) tests Message-ID: <5c09558e.1c69fb81.38a33.b8f0@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95429:3b0149d37897 Date: 2018-12-06 16:55 +0000 http://bitbucket.org/pypy/pypy/changeset/3b0149d37897/ Log: Convert test_grp to regular (app-level) tests diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py --- a/pypy/module/test_lib_pypy/test_grp_extra.py +++ b/pypy/module/test_lib_pypy/test_grp_extra.py @@ -1,42 +1,38 @@ -from pypy.module.test_lib_pypy.support import import_lib_pypy +import pytest +grp = pytest.importorskip('grp') -class AppTestGrp: - spaceconfig = dict(usemodules=('binascii', '_rawffi', 'itertools')) - def setup_class(cls): - cls.w_grp = import_lib_pypy(cls.space, 'grp', - "No grp module on this platform") +def test_basic(): + with pytest.raises(KeyError) as e: + grp.getgrnam("dEkLofcG") + assert e.value.args[0] == "'getgrnam(): name not found: dEkLofcG'" + for name in ["root", "wheel"]: + try: + g = grp.getgrnam(name) + except KeyError: + continue + assert g.gr_gid == 0 + assert 'root' in g.gr_mem or g.gr_mem == [] + assert g.gr_name == name + assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) + g2 = grp.getgrnam(unicode(name)) + assert g2 == g + break + else: + raise - def test_basic(self): - e = raises(KeyError, self.grp.getgrnam, "dEkLofcG") - assert e.value.args[0] == "'getgrnam(): name not found: dEkLofcG'" - for name in ["root", "wheel"]: - try: - g = self.grp.getgrnam(name) - except KeyError: - continue - assert g.gr_gid == 0 - assert 'root' in g.gr_mem or g.gr_mem == [] - assert g.gr_name == name - assert isinstance(g.gr_passwd, str) # usually just 'x', don't hope :-) - g2 = self.grp.getgrnam(unicode(name)) - assert g2 == g - break - else: - raise +def test_extra(): + with pytest.raises(TypeError): + grp.getgrnam(False) + with pytest.raises(TypeError): + grp.getgrnam(None) - def test_extra(self): - grp = self.grp - print(grp.__file__) - raises(TypeError, grp.getgrnam, False) - raises(TypeError, grp.getgrnam, None) - - def test_struct_group(self): - g = self.grp.struct_group((10, 20, 30, 40)) - assert len(g) == 4 - assert list(g) == [10, 20, 30, 40] - assert g.gr_name == 10 - assert g.gr_passwd == 20 - assert g.gr_gid == 30 - assert g.gr_mem == 40 +def test_struct_group(): + g = grp.struct_group((10, 20, 30, 40)) + assert len(g) == 4 + assert list(g) == [10, 20, 30, 40] + assert g.gr_name == 10 + assert g.gr_passwd == 20 + assert g.gr_gid == 30 + assert g.gr_mem == 40 From pypy.commits at gmail.com Thu Dec 6 12:00:01 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 06 Dec 2018 09:00:01 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_grp to extra_tests/ Message-ID: <5c095591.1c69fb81.e1150.2fb7@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95430:1f22f0021435 Date: 2018-12-06 16:59 +0000 http://bitbucket.org/pypy/pypy/changeset/1f22f0021435/ Log: Move test_grp to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_grp_extra.py b/extra_tests/test_grp.py rename from pypy/module/test_lib_pypy/test_grp_extra.py rename to extra_tests/test_grp.py From pypy.commits at gmail.com Thu Dec 6 12:32:20 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 06 Dec 2018 09:32:20 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Convert test_itertools to regular (app-level) tests; the issue on CPython was fixed long ago Message-ID: <5c095d24.1c69fb81.d9b67.7fa5@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95431:213df22b47c7 Date: 2018-12-06 17:30 +0000 http://bitbucket.org/pypy/pypy/changeset/213df22b47c7/ Log: Convert test_itertools to regular (app-level) tests; the issue on CPython was fixed long ago diff --git a/pypy/module/test_lib_pypy/test_itertools.py b/pypy/module/test_lib_pypy/test_itertools.py --- a/pypy/module/test_lib_pypy/test_itertools.py +++ b/pypy/module/test_lib_pypy/test_itertools.py @@ -1,22 +1,15 @@ -class AppTestItertools: - spaceconfig = dict(usemodules=['itertools']) +import sys +import itertools - def setup_class(cls): - cls.w_itertools = cls.space.appexec([], "(): import itertools; return itertools") +def test_chain(): + it = itertools.chain([], [1, 2, 3]) + lst = list(it) + assert lst == [1, 2, 3] - def test_chain(self): - it = self.itertools.chain([], [1, 2, 3]) - lst = list(it) - assert lst == [1, 2, 3] +def test_islice_maxint(): + slic = itertools.islice(itertools.count(), 1, 10, sys.maxint) + assert len(list(slic)) == 1 - def test_islice(self): - import sys - itertools = self.itertools - - slic = itertools.islice(itertools.count(), 1, 10, sys.maxint) - assert len(list(slic)) == 1 - - if '__pypy__' not in sys.builtin_module_names: - skip("this takes ages on top of CPython's itertools module") - slic = itertools.islice(itertools.count(), 1, 10, sys.maxint-20) - assert len(list(slic)) == 1 +def test_islice_largeint(): + slic = itertools.islice(itertools.count(), 1, 10, sys.maxint - 20) + assert len(list(slic)) == 1 From pypy.commits at gmail.com Thu Dec 6 12:32:22 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 06 Dec 2018 09:32:22 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_itertools to extra_tests/ Message-ID: <5c095d26.1c69fb81.c122f.8796@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95432:42c55db86dda Date: 2018-12-06 17:31 +0000 http://bitbucket.org/pypy/pypy/changeset/42c55db86dda/ Log: Move test_itertools to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_itertools.py b/extra_tests/test_itertools.py rename from pypy/module/test_lib_pypy/test_itertools.py rename to extra_tests/test_itertools.py From pypy.commits at gmail.com Fri Dec 7 03:47:11 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 07 Dec 2018 00:47:11 -0800 (PST) Subject: [pypy-commit] pypy default: A failing test for issue #2926 Message-ID: <5c0a338f.1c69fb81.38f8f.5f29@mx.google.com> Author: Armin Rigo Branch: Changeset: r95434:708fbffab6a0 Date: 2018-12-07 10:46 +0200 http://bitbucket.org/pypy/pypy/changeset/708fbffab6a0/ Log: A failing test for issue #2926 diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4758,3 +4758,56 @@ res = self.meta_interp(f, [2, 200]) assert res == f(2, 200) + + def test_issue2926(self): + driver = JitDriver(greens = [], reds=['i', 'total', 'p']) + + class Base(object): + def do_stuff(self): + return 1000 + class Int(Base): + def __init__(self, intval): + self.intval = intval + def do_stuff(self): + return self.intval + class SubInt(Int): + pass + class Float(Base): + def __init__(self, floatval): + self.floatval = floatval + def do_stuff(self): + return int(self.floatval) + + prebuilt = [Int(i) for i in range(10)] + + @dont_look_inside + def forget_intbounds(i): + return i + + @dont_look_inside + def escape(p): + pass + + def f(i): + total = 0 + p = Base() + while True: + driver.jit_merge_point(i=i, total=total, p=p) + #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i + if i == 13: + break + total += p.do_stuff() + j = forget_intbounds(i) + if j < 10: # initial loop + p = prebuilt[i] + p.intval = j + elif j < 12: + p = Int(i) + else: + p = Float(3.14) + escape(p) + i += 1 + return total + + res = self.meta_interp(f, [0]) + assert res == f(0) From pypy.commits at gmail.com Fri Dec 7 04:20:35 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 07 Dec 2018 01:20:35 -0800 (PST) Subject: [pypy-commit] pypy default: Fix for 708fbffab6a0 Message-ID: <5c0a3b63.1c69fb81.d977c.f67d@mx.google.com> Author: Armin Rigo Branch: Changeset: r95435:b010806d20bf Date: 2018-12-07 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/b010806d20bf/ Log: Fix for 708fbffab6a0 diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -389,6 +389,8 @@ def optimize_GUARD_SUBCLASS(self, op): info = self.getptrinfo(op.getarg(0)) optimizer = self.optimizer + # must raise 'InvalidLoop' in all cases where 'info' shows the + # class cannot possibly match (see test_issue2926) if info and info.is_constant(): c = self.get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.ts.cls_of_box(c).getint() @@ -398,13 +400,29 @@ if info is not None and info.is_about_object(): known_class = info.get_known_class(optimizer.cpu) if known_class: + # Class of 'info' is exactly 'known_class'. + # We know statically if the 'guard_subclass' will pass or fail. if optimizer._check_subclass(known_class.getint(), op.getarg(1).getint()): return + else: + raise InvalidLoop( + "GUARD_SUBCLASS(known_class) proven to always fail") elif info.get_descr() is not None: - if optimizer._check_subclass(info.get_descr().get_vtable(), + # Class of 'info' is either get_descr() or a subclass of it. + # We're keeping the 'guard_subclass' at runtime only in the + # case where get_descr() is some strict parent class of + # the argument to 'guard_subclass'. + info_base_descr = info.get_descr().get_vtable() + if optimizer._check_subclass(info_base_descr, op.getarg(1).getint()): - return + return # guard_subclass always passing + elif optimizer._check_subclass(op.getarg(1).getint(), + info_base_descr): + pass # don't know, must keep the 'guard_subclass' + else: + raise InvalidLoop( + "GUARD_SUBCLASS(base_class) proven to always fail") return self.emit(op) def optimize_GUARD_NONNULL(self, op): From pypy.commits at gmail.com Fri Dec 7 04:34:53 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 07 Dec 2018 01:34:53 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: Review the low-level parts. Fixes some details Message-ID: <5c0a3ebd.1c69fb81.c4ad9.52a3@mx.google.com> Author: Armin Rigo Branch: math-improvements Changeset: r95436:ecb5ca585871 Date: 2018-12-07 11:34 +0200 http://bitbucket.org/pypy/pypy/changeset/ecb5ca585871/ Log: Review the low-level parts. Fixes some details diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -325,8 +325,6 @@ 'lllong_xor': LLOp(canfold=True), 'ulllong_is_true': LLOp(canfold=True), - 'ulllong_neg': LLOp(canfold=True), - 'ulllong_abs': LLOp(canfold=True), 'ulllong_invert': LLOp(canfold=True), 'ulllong_add': LLOp(canfold=True), diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -538,9 +538,8 @@ if name.startswith('unsigned'): name = 'u' + name[9:] signed = False - elif name.startswith('__u'): - signed = False - elif name == 'size_t' or name.startswith('uint'): + elif (name == 'size_t' or name.startswith('uint') + or name.startswith('__uint')): signed = False else: signed = True diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py --- a/rpython/rtyper/rint.py +++ b/rpython/rtyper/rint.py @@ -188,7 +188,7 @@ signedlonglonglong_repr = getintegerrepr(SignedLongLongLong, 'lllong_') unsigned_repr = getintegerrepr(Unsigned, 'uint_') unsignedlonglong_repr = getintegerrepr(UnsignedLongLong, 'ullong_') -signedlonglonglong_repr = getintegerrepr(UnsignedLongLongLong, 'ulllong_') +unsignedlonglonglong_repr = getintegerrepr(UnsignedLongLongLong, 'ulllong_') class __extend__(pairtype(IntegerRepr, IntegerRepr)): @@ -565,7 +565,7 @@ @jit.dont_look_inside def ll_ulllong_py_mod(x, y): - return llop.ullong_mod(UnsignedLongLongLong, x, y) + return llop.ulllong_mod(UnsignedLongLongLong, x, y) def ll_ulllong_py_mod_zer(x, y): if y == 0: diff --git a/rpython/translator/c/src/int.h b/rpython/translator/c/src/int.h --- a/rpython/translator/c/src/int.h +++ b/rpython/translator/c/src/int.h @@ -258,18 +258,18 @@ #define OP_ULLONG_OR OP_LLONG_OR #define OP_ULLONG_XOR OP_LLONG_XOR -#define OP_ULLLONG_IS_TRUE OP_LLONG_IS_TRUE -#define OP_ULLLONG_INVERT OP_LLONG_INVERT -#define OP_ULLLONG_ADD OP_LLONG_ADD -#define OP_ULLLONG_SUB OP_LLONG_SUB -#define OP_ULLLONG_MUL OP_LLONG_MUL -#define OP_ULLLONG_LT OP_LLONG_LT -#define OP_ULLLONG_LE OP_LLONG_LE -#define OP_ULLLONG_EQ OP_LLONG_EQ -#define OP_ULLLONG_NE OP_LLONG_NE -#define OP_ULLLONG_GT OP_LLONG_GT -#define OP_ULLLONG_GE OP_LLONG_GE -#define OP_ULLLONG_AND OP_LLONG_AND -#define OP_ULLLONG_OR OP_LLONG_OR -#define OP_ULLLONG_XOR OP_LLONG_XOR +#define OP_ULLLONG_IS_TRUE OP_LLLONG_IS_TRUE +#define OP_ULLLONG_INVERT OP_LLLONG_INVERT +#define OP_ULLLONG_ADD OP_LLLONG_ADD +#define OP_ULLLONG_SUB OP_LLLONG_SUB +#define OP_ULLLONG_MUL OP_LLLONG_MUL +#define OP_ULLLONG_LT OP_LLLONG_LT +#define OP_ULLLONG_LE OP_LLLONG_LE +#define OP_ULLLONG_EQ OP_LLLONG_EQ +#define OP_ULLLONG_NE OP_LLLONG_NE +#define OP_ULLLONG_GT OP_LLLONG_GT +#define OP_ULLLONG_GE OP_LLLONG_GE +#define OP_ULLLONG_AND OP_LLLONG_AND +#define OP_ULLLONG_OR OP_LLLONG_OR +#define OP_ULLLONG_XOR OP_LLLONG_XOR From pypy.commits at gmail.com Sat Dec 8 01:39:11 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 07 Dec 2018 22:39:11 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Use a fixture to inject the test dll; random cleanups Message-ID: <5c0b670f.1c69fb81.7b2a5.3c40@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95439:9bf67667765c Date: 2018-12-08 06:38 +0000 http://bitbucket.org/pypy/pypy/changeset/9bf67667765c/ Log: Use a fixture to inject the test dll; random cleanups diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/pypy/module/test_lib_pypy/ctypes_tests/conftest.py --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/conftest.py @@ -96,8 +96,11 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) -# we need to run after the "tmpdir" plugin which installs pytest.ensuretemp - at pytest.mark.trylast -def pytest_configure(config): - global sofile - sofile = compile_so_file() + at pytest.fixture(scope='session') +def sofile(): + return str(compile_so_file()) + + at pytest.fixture +def dll(sofile): + from ctypes import CDLL + return CDLL(str(sofile)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py @@ -145,11 +145,7 @@ assert res == 1111 assert rect.left == -1000 # must not have been changed! - def test_callback_from_c_with_struct_argument(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - + def test_callback_from_c_with_struct_argument(self, dll): class RECT(Structure): _fields_ = [("left", c_long), ("top", c_long), ("right", c_long), ("bottom", c_long)] @@ -177,11 +173,7 @@ proto(lambda r: 0) - def test_qsort(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - + def test_qsort(self, dll): PI = POINTER(c_int) A = c_int*5 a = A() @@ -205,11 +197,7 @@ assert res == [1,2,3,4,5] - def test_pyobject_as_opaque(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - + def test_pyobject_as_opaque(self, dll): def callback(arg): return arg() @@ -220,11 +208,7 @@ res = cfunc(CTP(callback), lambda : 3) assert res == 3 - def test_callback_void(self, capsys): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - + def test_callback_void(self, capsys, dll): def callback(): pass diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py @@ -2,15 +2,11 @@ import sys, py from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - mod.lib = CDLL(str(conftest.sofile)) - class TestCast(BaseCTypesTestChecker): - def test_cast_functype(self): + def test_cast_functype(self, dll): # make sure we can cast function type - my_sqrt = lib.my_sqrt + my_sqrt = dll.my_sqrt saved_objects = my_sqrt._objects.copy() sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) assert sqrt(4.0) == 2.0 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_checkretval.py b/pypy/module/test_lib_pypy/ctypes_tests/test_checkretval.py deleted file mode 100644 --- a/pypy/module/test_lib_pypy/ctypes_tests/test_checkretval.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest -import sys - -from ctypes import * - -def setup_module(mod): - import conftest - mod.dll = CDLL(str(conftest.sofile)) - -class CHECKED(c_int): - def _check_retval_(value): - # Receives a CHECKED instance. - return str(value.value) - _check_retval_ = staticmethod(_check_retval_) - -class TestRetval: - - def test_checkretval(self): - assert 42 == dll._testfunc_p_p(42) - - dll._testfunc_p_p.restype = CHECKED - assert "42" == dll._testfunc_p_p(42) - - dll._testfunc_p_p.restype = None - assert None == dll._testfunc_p_p(42) - - del dll._testfunc_p_p.restype - assert 42 == dll._testfunc_p_p(42) - - try: - oledll - except NameError: - pass - else: - def test_oledll(self): - with pytest.raises(WindowsError): - oledll.oleaut32.CreateTypeLib2(0, 0, 0) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py @@ -1,12 +1,7 @@ -import py +import pytest import ctypes -from _ctypes import function - -try: - import _rawffi -except ImportError: - py.test.skip("app-level test only for PyPy") +_rawffi = pytest.importorskip('_rawffi') # PyPy-only class TestErrno: @@ -15,7 +10,7 @@ assert _rawffi.get_errno() == 42 assert ctypes.get_errno() == old check.free_temp_buffers = lambda *args: None - f = function.CFuncPtr() + f = ctypes._CFuncPtr() old = _rawffi.get_errno() f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO ctypes.set_errno(42) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py @@ -188,10 +188,7 @@ x = (c_int * 42)() assert sizeof(x) == 42 * sizeof(c_int) - def test_convert_pointers(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + def test_convert_pointers(self, dll): func = dll._testfunc_p_p func.restype = c_char_p diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py b/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py @@ -1,12 +1,6 @@ from ctypes import * -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - mod.lib = CDLL(_ctypes_test) - - class TestCFuncPtr: - def test_restype(self): - foo = lib.my_unused_function + def test_restype(self, dll): + foo = dll.my_unused_function assert foo.restype is c_int # by default diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py @@ -1,32 +1,22 @@ from ctypes import * import sys import pytest -from .support import BaseCTypesTestChecker -try: - WINFUNCTYPE -except NameError: - # fake to enable this test on Linux - WINFUNCTYPE = CFUNCTYPE + at pytest.fixture +def dll(sofile): + return CDLL(str(sofile), use_errno=True) -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - mod.dll = CDLL(_ctypes_test, use_errno=True) - if sys.platform == "win32": - mod.windll = WinDLL(_ctypes_test) +class TestFunctions(object): -class TestFunctions(BaseCTypesTestChecker): - - def test_char_result(self): + def test_char_result(self, dll): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_char result = f(0, 0, 0, 0, 0, 0) assert result == '\x00' - def test_boolresult(self): + def test_boolresult(self, dll): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_bool @@ -35,14 +25,14 @@ true_result = f(1, 0, 0, 0, 0, 0) assert true_result is True - def test_unicode_function_name(self): + def test_unicode_function_name(self, dll): f = dll[u'_testfunc_i_bhilfd'] f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_int result = f(1, 2, 3, 4, 5.0, 6.0) assert result == 21 - def test_truncate_python_longs(self): + def test_truncate_python_longs(self, dll): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_int @@ -50,8 +40,7 @@ result = f(x, x, x, x, 0, 0) assert result == -8 - - def test_convert_pointers(self): + def test_convert_pointers(self, dll): f = dll.deref_LP_c_char_p f.restype = c_char f.argtypes = [POINTER(c_char_p)] @@ -63,15 +52,14 @@ ################################################################ - - def test_call_some_args(self): + def test_call_some_args(self, dll): f = dll.my_strchr f.argtypes = [c_char_p] f.restype = c_char_p result = f("abcd", ord("b")) assert result == "bcd" - def test_keepalive_buffers(self, monkeypatch): + def test_keepalive_buffers(self, monkeypatch, dll): import gc f = dll.my_strchr f.argtypes = [c_char_p] @@ -88,7 +76,7 @@ result = f("abcd", ord("b")) assert result == "bcd" - def test_caching_bug_1(self): + def test_caching_bug_1(self, dll): # the same test as test_call_some_args, with two extra lines # in the middle that trigger caching in f._ptr, which then # makes the last two lines fail @@ -100,7 +88,7 @@ result = f("abcd", ord("b"), 42) assert result == "bcd" - def test_argument_conversion_and_checks(self): + def test_argument_conversion_and_checks(self, dll): #This test is designed to check for segfaults if the wrong type of argument is passed as parameter strlen = dll.my_strchr strlen.argtypes = [c_char_p, c_int] @@ -113,7 +101,7 @@ with pytest.raises(ArgumentError): strlen(False, 0) - def test_union_as_passed_value(self): + def test_union_as_passed_value(self, dll): class UN(Union): _fields_ = [("x", c_short), ("y", c_long)] @@ -123,9 +111,9 @@ a = A() a[1].x = 33 u = dll.ret_un_func(a[1]) - assert u.y == 33*10000 + assert u.y == 33 * 10000 - def test_cache_funcptr(self): + def test_cache_funcptr(self, dll): tf_b = dll.tf_b tf_b.restype = c_byte tf_b.argtypes = (c_byte,) @@ -135,7 +123,7 @@ assert tf_b(-126) == -42 assert tf_b._ptr is ptr - def test_custom_from_param(self): + def test_custom_from_param(self, dll): class A(c_byte): @classmethod def from_param(cls, obj): @@ -150,7 +138,7 @@ assert seen == ["yadda"] @pytest.mark.xfail(reason="warnings are disabled") - def test_warnings(self): + def test_warnings(self, dll): import warnings warnings.simplefilter("always") with warnings.catch_warnings(record=True) as w: @@ -160,7 +148,8 @@ assert "C function without declared arguments called" in str(w[0].message) @pytest.mark.xfail - def test_errcheck(self): + def test_errcheck(self, dll): + import warnings def errcheck(result, func, args): assert result == -42 assert type(result) is int @@ -189,8 +178,7 @@ warnings.resetwarnings() - - def test_errno(self): + def test_errno(self, dll): test_errno = dll.test_errno test_errno.restype = c_int set_errno(42) @@ -200,7 +188,7 @@ set_errno(0) assert get_errno() == 0 - def test_issue1655(self): + def test_issue1655(self, dll): def ret_list_p(icount): def sz_array_p(obj, func, args): assert ('.LP_c_int object' in repr(obj) or @@ -208,7 +196,7 @@ assert repr(args) in ("('testing!', c_int(4))", "('testing!', c_long(4))") assert args[icount].value == 4 - return [ obj[i] for i in range(args[icount].value) ] + return [obj[i] for i in range(args[icount].value)] return sz_array_p get_data_prototype = CFUNCTYPE(POINTER(c_int), @@ -216,7 +204,7 @@ get_data_paramflag = ((1,), (2,)) get_data_signature = ('test_issue1655', dll) - get_data = get_data_prototype( get_data_signature, get_data_paramflag ) + get_data = get_data_prototype(get_data_signature, get_data_paramflag) assert get_data('testing!') == 4 get_data.errcheck = ret_list_p(1) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py b/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py @@ -28,14 +28,12 @@ pass s = Stuff() s._as_parameter_ = None - + assert guess(s) == c_void_p -def test_guess_unicode(): +def test_guess_unicode(dll): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': py.test.skip("CPython segfaults: see http://bugs.python.org/issue5203") - import conftest - dll = CDLL(str(conftest.sofile)) wcslen = dll.my_wcslen text = u"Some long unicode string" assert wcslen(text) == len(text) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_loading.py b/pypy/module/test_lib_pypy/ctypes_tests/test_loading.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_loading.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_loading.py @@ -1,9 +1,5 @@ -import pytest -from ctypes import * -import sys -import os, StringIO +from ctypes import CDLL from ctypes.util import find_library -from ctypes.test import is_resource_enabled class TestLoader: diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py @@ -1,26 +1,6 @@ import pytest from ctypes import * from .support import BaseCTypesTestChecker -import sys, struct - -def valid_ranges(*types): - # given a sequence of numeric types, collect their _type_ - # attribute, which is a single format character compatible with - # the struct module, use the struct module to calculate the - # minimum and maximum value allowed for this format. - # Returns a list of (min, max) values. - result = [] - for t in types: - fmt = t._type_ - size = struct.calcsize(fmt) - a = struct.unpack(fmt, ("\x00"*32)[:size])[0] - b = struct.unpack(fmt, ("\xFF"*32)[:size])[0] - c = struct.unpack(fmt, ("\x7F"+"\x00"*32)[:size])[0] - d = struct.unpack(fmt, ("\x80"+"\xFF"*32)[:size])[0] - result.append((min(a, b, c, d), max(a, b, c, d))) - return result - -ArgType = type(byref(c_int(0))) unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong] signed_types = [c_byte, c_short, c_int, c_long, c_longlong] @@ -36,9 +16,6 @@ unsigned_types.append(c_ulonglong) signed_types.append(c_longlong) -unsigned_ranges = valid_ranges(*unsigned_types) -signed_ranges = valid_ranges(*signed_types) - ################################################################ class TestNumber(BaseCTypesTestChecker): diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py b/pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py @@ -1,6 +1,3 @@ -import pytest -import sys - class TestSimpleTypes: def test_pointer_subclasses(self): @@ -15,13 +12,10 @@ assert Void_pp.from_param(o) is o - def test_multiple_signature(self): + def test_multiple_signature(self, dll): # when .argtypes is not set, calling a function with a certain # set of parameters should not prevent another call with # another set. - from ctypes import CDLL, byref - import conftest - dll = CDLL(str(conftest.sofile)) func = dll._testfunc_p_p # This is call has too many arguments @@ -29,4 +23,3 @@ # This one is normal assert func(None) == 0 - diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py @@ -7,10 +7,6 @@ python_types = [int, int, int, int, int, long, int, long, long, long, float, float] -def setup_module(mod): - import conftest - mod._ctypes_test = str(conftest.sofile) - class TestPointers(BaseCTypesTestChecker): def test_get_ffi_argtype(self): @@ -36,8 +32,7 @@ assert p2.contents.contents.value == 42 assert p1.contents.value == 42 - def test_c_char_p_byref(self): - dll = CDLL(_ctypes_test) + def test_c_char_p_byref(self, dll): TwoOutArgs = dll.TwoOutArgs TwoOutArgs.restype = None TwoOutArgs.argtypes = [c_int, c_void_p, c_int, c_void_p] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py b/pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py @@ -2,20 +2,15 @@ from ctypes import * from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - mod.testdll = CDLL(_ctypes_test) - class TestFuncPrototypes(BaseCTypesTestChecker): - def test_restype_setattr(self): - func = testdll._testfunc_p_p + def test_restype_setattr(self, dll): + func = dll._testfunc_p_p with pytest.raises(TypeError): setattr(func, 'restype', 20) - def test_argtypes_setattr(self): - func = testdll._testfunc_p_p + def test_argtypes_setattr(self, dll): + func = dll._testfunc_p_p with pytest.raises(TypeError): setattr(func, 'argtypes', 20) with pytest.raises(TypeError): @@ -34,10 +29,10 @@ setattr(func, 'paramflags', ((1,), ('a',))) func.paramflags = (1,), (1|4,) - def test_kwargs(self): + def test_kwargs(self, dll): proto = CFUNCTYPE(c_char_p, c_char_p, c_int) paramflags = (1, 'text', "tavino"), (1, 'letter', ord('v')) - func = proto(('my_strchr', testdll), paramflags) + func = proto(('my_strchr', dll), paramflags) assert func.argtypes == (c_char_p, c_int) assert func.restype == c_char_p @@ -63,9 +58,9 @@ assert result == "possible" class TestArray(BaseCTypesTestChecker): - def test_array_to_ptr_wrongtype(self): + def test_array_to_ptr_wrongtype(self, dll): ARRAY = c_byte * 8 - func = testdll._testfunc_ai8 + func = dll._testfunc_ai8 func.restype = POINTER(c_int) func.argtypes = [c_int * 8] array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_unions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_unions.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_unions.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_unions.py @@ -28,4 +28,3 @@ u = UnionofStuff() u.one.x = 3 assert u.two.x == 3 - diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_values.py b/pypy/module/test_lib_pypy/ctypes_tests/test_values.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_values.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_values.py @@ -5,15 +5,10 @@ from ctypes import * from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - mod.ctdll = CDLL(_ctypes_test) - class TestValues(BaseCTypesTestChecker): - def test_a_string(self): - a_string = (c_char * 16).in_dll(ctdll, "a_string") + def test_a_string(self, dll): + a_string = (c_char * 16).in_dll(dll, "a_string") assert a_string.raw == "0123456789abcdef" a_string[15] = '$' - assert ctdll.get_a_string_char(15) == ord('$') + assert dll.get_a_string_char(15) == ord('$') From pypy.commits at gmail.com Sun Dec 9 05:22:00 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:00 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix maketrans Message-ID: <5c0cecc8.1c69fb81.bac5a.6a69@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95440:cf6ed65cb1da Date: 2018-12-02 13:03 -0800 http://bitbucket.org/pypy/pypy/changeset/cf6ed65cb1da/ Log: test, fix maketrans diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -636,6 +636,8 @@ assert 'c' == 'abababc'.translate(tbl) tbl = str.maketrans('abc', 'xyz', 'd') assert 'xyzzy' == 'abdcdcbdddd'.translate(tbl) + tbl = str.maketrans({'\xe9': 'a'}) + assert "[\xe9]".translate(tbl) == "[a]" raises(TypeError, str.maketrans) raises(ValueError, str.maketrans, 'abc', 'defg') diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -259,12 +259,12 @@ w_key, w_value = space.unpackiterable(w_item, 2) if space.isinstance_w(w_key, space.w_unicode): # convert string keys to integer keys - key = space.utf8_w(w_key) - if len(key) != 1: + if space.len_w(w_key) != 1: raise oefmt(space.w_ValueError, "string keys in translate table must be " "of length 1") - w_key = space.newint(ord(key[0])) + val = space.utf8_w(w_key) + w_key = space.newint(rutf8.codepoint_at_pos(val, 0)) else: # just keep integer keys try: From pypy.commits at gmail.com Sun Dec 9 05:22:02 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:02 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix formatting '%s.2' for unicode Message-ID: <5c0cecca.1c69fb81.3654b.e5d6@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95441:e0b40146105a Date: 2018-12-02 14:08 -0800 http://bitbucket.org/pypy/pypy/changeset/e0b40146105a/ Log: test, fix formatting '%s.2' for unicode diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -335,7 +335,8 @@ @specialize.arg(2) def std_wp(self, r, is_string=False): - length = len(r) + # r is utf8-encoded unicode + length = rutf8.codepoints_in_utf8(r) if do_unicode and is_string: # convert string to unicode using the default encoding r = self.space.utf8_w(self.space.newbytes(r)) @@ -346,6 +347,10 @@ return if prec >= 0 and prec < length: length = prec # ignore the end of the string if too long + if do_unicode: + # XXX could use W_UnicodeObject.descr_getslice, but that would + # require a refactor to use the w_val, not r + length = rutf8._pos_at_index(r, length) result = self.result padding = self.width - length if padding < 0: diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -1002,7 +1002,8 @@ def test_formatting_uchr(self): assert '%c' % '\U00021483' == '\U00021483' - def test_formatting_unicode__str__(self): + def test_formatting_unicode__str__0(self): + assert '%.2s' % "a\xe9\u20ac" == 'a\xe9' class A: def __init__(self, num): self.num = num From pypy.commits at gmail.com Sun Dec 9 05:22:04 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:04 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add failing test Message-ID: <5c0ceccc.1c69fb81.9f10b.15cd@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95442:987e7f0a2866 Date: 2018-12-02 18:48 -0800 http://bitbucket.org/pypy/pypy/changeset/987e7f0a2866/ Log: add failing test diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -148,6 +148,98 @@ lgt = 12 assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', lgt) + def test_unicode_replace(self): + # CPython #8271: during the decoding of an invalid UTF-8 byte sequence, + # only the start byte and the continuation byte(s) are now considered + # invalid, instead of the number of bytes specified by the start byte. + # See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 42, + # table 3-8, Row 2) for more information about the algorithm used. + FFFD = '\ufffd' + sequences = [ + # invalid start bytes + (b'\x80', FFFD), # continuation byte + (b'\x80\x80', FFFD*2), # 2 continuation bytes + (b'\xc0', FFFD), + (b'\xc0\xc0', FFFD*2), + (b'\xc1', FFFD), + (b'\xc1\xc0', FFFD*2), + (b'\xc0\xc1', FFFD*2), + # with start byte of a 2-byte sequence + (b'\xc2', FFFD), # only the start byte + (b'\xc2\xc2', FFFD*2), # 2 start bytes + (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes + (b'\xc2\x41', FFFD+'A'), # invalid continuation byte + # with start byte of a 3-byte sequence + (b'\xe1', FFFD), # only the start byte + (b'\xe1\xe1', FFFD*2), # 2 start bytes + (b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes + (b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes + (b'\xe1\x80', FFFD), # only 1 continuation byte + (b'\xe1\x41', FFFD+'A'), # invalid continuation byte + (b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb + (b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes + (b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte + (b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid + (b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid + # with start byte of a 4-byte sequence + (b'\xf1', FFFD), # only the start byte + (b'\xf1\xf1', FFFD*2), # 2 start bytes + (b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes + (b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes + (b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes + (b'\xf1\x80', FFFD), # only 1 continuation bytes + (b'\xf1\x80\x80', FFFD), # only 2 continuation bytes + (b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid + (b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid + (b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid + (b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid + (b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid + (b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid + (b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid + (b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD), + (b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2), + (b'\xf1\xf1\x80\x41', FFFD*2+'A'), + (b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2), + # with invalid start byte of a 4-byte sequence (rfc2279) + (b'\xf5', FFFD), # only the start byte + (b'\xf5\xf5', FFFD*2), # 2 start bytes + (b'\xf5\x80', FFFD*2), # only 1 continuation byte + (b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte + (b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes + (b'\xf5\x80\x41', FFFD*2+'A'), # 1 valid cb and 1 invalid + (b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD), + (b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'), + # with invalid start byte of a 5-byte sequence (rfc2279) + (b'\xf8', FFFD), # only the start byte + (b'\xf8\xf8', FFFD*2), # 2 start bytes + (b'\xf8\x80', FFFD*2), # only one continuation byte + (b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid + (b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes + # with invalid start byte of a 6-byte sequence (rfc2279) + (b'\xfc', FFFD), # only the start byte + (b'\xfc\xfc', FFFD*2), # 2 start bytes + (b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes + (b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes + # invalid start byte + (b'\xfe', FFFD), + (b'\xfe\x80\x80', FFFD*3), + # other sequences + (b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'), + (b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'), + (b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'), + (b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64', + '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'), + ] + for n, (seq, res) in enumerate(sequences): + print([hex(x) for x in seq], [hex(ord(x)) for x in res]) + raises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict') + uni = seq.decode('utf-8', 'replace') + assert uni == res + uni = (seq+b'b').decode('utf-8', 'replace') + assert uni == res+'b' + uni = seq.decode('utf-8', 'ignore') + assert uni == res.replace('\uFFFD', '') + def test_unexpected_end_of_data(self): """ Test that an 'unexpected end of data' error is raised when the string @@ -171,7 +263,6 @@ ] FFFD = '\ufffd' for seq in sequences: - print(seq) bseq = bytes(int(c, 16) for c in seq.split()) exc = raises(UnicodeDecodeError, bseq.decode, 'utf-8') assert 'unexpected end of data' in str(exc.value) @@ -413,7 +504,6 @@ def search_function(encoding): def f(input, errors="strict"): return 42 - print(encoding) if encoding == 'test.mytestenc': return (f, f, None, None) return None @@ -871,7 +961,6 @@ encodings = ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-32-le', 'utf-32-be') for encoding in encodings: - print('encoding', encoding) raises(UnicodeEncodeError, u'\ud800'.encode, encoding) assert (u'[\udc80]'.encode(encoding, "backslashreplace") == '[\\udc80]'.encode(encoding)) @@ -884,7 +973,6 @@ ('utf-16-be', b'\xdc\x80'), ('utf-32-le', b'\x80\xdc\x00\x00'), ('utf-32-be', b'\x00\x00\xdc\x80')]: - print(encoding) before, after = "[", "]" before_sequence = before.encode(encoding) after_sequence = after.encode(encoding) @@ -1053,7 +1141,6 @@ assert w[0].category == DeprecationWarning with warnings.catch_warnings(record=True) as w: - print(type(encoded_abc)) decoder(encoded_abc) assert len(w) == 1 assert str(w[0].message) == warning_msg From pypy.commits at gmail.com Sun Dec 9 05:22:06 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:06 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix for 'replace' error handler and short sequences Message-ID: <5c0cecce.1c69fb81.1f532.ece1@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95443:0cca4bcffdbf Date: 2018-12-02 20:56 -0800 http://bitbucket.org/pypy/pypy/changeset/0cca4bcffdbf/ Log: test, fix for 'replace' error handler and short sequences diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -349,18 +349,21 @@ res = StringBuilder(slen) pos = 0 end = len(s) + suppressing = False # we are in a chain of "bad" unicode, only emit one fix while pos < end: ordch1 = ord(s[pos]) # fast path for ASCII if ordch1 <= 0x7F: pos += 1 res.append(chr(ordch1)) + suppressing = False continue if ordch1 <= 0xC1: r, pos, rettype = errorhandler(errors, "utf8", "invalid start byte", s, pos, pos + 1) - res.append(r) + if not suppressing: + res.append(r) continue pos += 1 @@ -372,14 +375,16 @@ break r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", s, pos - 1, pos) - res.append(r) + if not suppressing: + res.append(r) continue ordch2 = ord(s[pos]) if rutf8._invalid_byte_2_of_2(ordch2): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) - res.append(r) + if not suppressing: + res.append(r) continue # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz pos += 1 @@ -393,8 +398,9 @@ pos -= 1 break r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", - s, pos - 1, pos + 1) + s, pos - 1, pos) res.append(r) + suppressing = True continue ordch2 = ord(s[pos]) ordch3 = ord(s[pos + 1]) @@ -402,12 +408,14 @@ if rutf8._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) - res.append(r) + if not suppressing: + res.append(r) continue elif rutf8._invalid_byte_3_of_3(ordch3): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos + 1) - res.append(r) + if not suppressing: + res.append(r) continue pos += 2 @@ -415,6 +423,7 @@ res.append(chr(ordch1)) res.append(chr(ordch2)) res.append(chr(ordch3)) + suppressing = False continue if ordch1 <= 0xF4: @@ -424,6 +433,8 @@ break r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", s, pos - 1, pos) + res.append(r) + suppressing = True continue ordch2 = ord(s[pos]) ordch3 = ord(s[pos + 1]) @@ -432,7 +443,8 @@ if rutf8._invalid_byte_2_of_4(ordch1, ordch2): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) - res.append(r) + if not suppressing: + res.append(r) continue elif rutf8._invalid_byte_3_of_4(ordch3): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", @@ -442,7 +454,8 @@ elif rutf8._invalid_byte_4_of_4(ordch4): r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos + 2) - res.append(r) + if not suppressing: + res.append(r) continue pos += 3 @@ -451,11 +464,13 @@ res.append(chr(ordch2)) res.append(chr(ordch3)) res.append(chr(ordch4)) + suppressing = False continue r, pos, rettype = errorhandler(errors, "utf8", "invalid start byte", s, pos - 1, pos) - res.append(r) + if not suppressing: + res.append(r) r = res.build() return r, rutf8.check_utf8(r, True), pos diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -261,12 +261,12 @@ 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF', 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF' ] - FFFD = '\ufffd' for seq in sequences: bseq = bytes(int(c, 16) for c in seq.split()) exc = raises(UnicodeDecodeError, bseq.decode, 'utf-8') assert 'unexpected end of data' in str(exc.value) - assert bseq.decode('utf-8', 'replace') == u'\ufffd' + useq = bseq.decode('utf-8', 'replace') + assert useq == u'\ufffd', (bseq, useq) assert ((b'aaaa' + bseq + b'bbbb').decode('utf-8', 'replace') == u'aaaa\ufffdbbbb') assert bseq.decode('utf-8', 'ignore') == '' From pypy.commits at gmail.com Sun Dec 9 05:22:07 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:07 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: cleanup Message-ID: <5c0ceccf.1c69fb81.28312.e87d@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95444:c245f74af234 Date: 2018-12-07 12:06 +0200 http://bitbucket.org/pypy/pypy/changeset/c245f74af234/ Log: cleanup diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -231,7 +231,6 @@ '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'), ] for n, (seq, res) in enumerate(sequences): - print([hex(x) for x in seq], [hex(ord(x)) for x in res]) raises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict') uni = seq.decode('utf-8', 'replace') assert uni == res @@ -272,7 +271,7 @@ assert bseq.decode('utf-8', 'ignore') == '' assert ((b'aaaa' + bseq + b'bbbb').decode('utf-8', 'ignore') == u'aaaabbbb') - + class AppTestPartialEvaluation: spaceconfig = dict(usemodules=['array',]) From pypy.commits at gmail.com Sun Dec 9 05:22:11 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:11 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5c0cecd3.1c69fb81.f1777.5898@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95445:5c55aa22365a Date: 2018-12-09 08:39 +0200 http://bitbucket.org/pypy/pypy/changeset/5c55aa22365a/ Log: merge default into branch diff too long, truncating to 2000 out of 2760 lines diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -182,6 +182,8 @@ module = category[:i] klass = category[i+1:] try: + if not module: + raise ImportError # instead of the ValueError we'd get m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -55,3 +55,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -609,8 +609,14 @@ warnoptions.extend(pythonwarnings.split(',')) if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -977,6 +977,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -422,6 +422,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -371,10 +371,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -403,28 +407,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -211,9 +211,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -576,6 +576,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in e.value.message + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in excinfo.value.message - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in e.value.message +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in excinfo.value.message - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py From pypy.commits at gmail.com Sun Dec 9 05:22:13 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:13 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5c0cecd5.1c69fb81.d006d.c611@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95446:1300ca1763e1 Date: 2018-12-09 11:13 +0200 http://bitbucket.org/pypy/pypy/changeset/1300ca1763e1/ Log: merge default into branch diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy", allow_module_level=True) + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -51,3 +51,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -386,7 +386,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -603,8 +603,14 @@ warnoptions = pythonwarnings.split(',') + warnoptions if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in str(e.value) + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in str(excinfo.value) - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in str(e.value) +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in str(excinfo.value) - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -44,7 +44,7 @@ if config.translation.jit_profiler == "oprofile": from rpython.jit.backend.x86 import oprofile if not oprofile.OPROFILE_AVAILABLE: - log.WARNING('oprofile support was explicitly enabled, but oprofile headers seem not to be available') + raise Exception('oprofile support was explicitly enabled, but oprofile headers seem not to be available') profile_agent = oprofile.OProfileAgent() self.with_threads = config.translation.thread diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -11,7 +11,7 @@ from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - AbstractResOp, GuardResOp + GuardResOp from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -6,7 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rlib.rarithmetic import intmask @@ -44,8 +44,9 @@ if b.has_lower and b.has_upper and b.lower == b.upper: self.make_constant_int(box, b.lower) - if isinstance(box, AbstractResOp): - dispatch_bounds_ops(self, box) + box1 = self.optimizer.as_operation(box) + if box1 is not None: + dispatch_bounds_ops(self, box1) def _optimize_guard_true_false_value(self, op): return self.emit(op) @@ -126,10 +127,11 @@ v1, v2 = v2, v1 # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): - if not self.optimizer.is_inputarg(arg1): + arg1 = self.optimizer.as_operation(arg1) + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = arg1.getarg(0) - prod_arg2 = arg1.getarg(1) + prod_arg1 = self.get_box_replacement(arg1.getarg(0)) + prod_arg2 = self.get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -279,6 +279,7 @@ self.quasi_immutable_deps = None self.replaces_guard = {} self._newoperations = [] + self._emittedoperations = {} self.optimizer = self self.optpure = None self.optheap = None @@ -292,11 +293,6 @@ self.set_optimizations(optimizations) self.setup() - def init_inparg_dict_from(self, lst): - self.inparg_dict = {} - for box in lst: - self.inparg_dict[box] = None - def set_optimizations(self, optimizations): if optimizations: self.first_optimization = optimizations[0] @@ -384,9 +380,12 @@ return info.force_box(op, optforce) return op - def is_inputarg(self, op): - return True - return op in self.inparg_dict + def as_operation(self, op): + # You should never check "isinstance(op, AbstractResOp" directly. + # Instead, use this helper. + if isinstance(op, AbstractResOp) and op in self._emittedoperations: + return op + return None def get_constant_box(self, box): box = self.get_box_replacement(box) @@ -404,6 +403,7 @@ def clear_newoperations(self): self._newoperations = [] + self._emittedoperations = {} def make_equal_to(self, op, newop): op = self.get_box_replacement(op) @@ -631,6 +631,7 @@ self._last_guard_op = None self._really_emitted_operation = op self._newoperations.append(op) + self._emittedoperations[op] = None def emit_guard_operation(self, op, pendingfields): guard_op = op # self.replace_op_with(op, op.getopnum()) @@ -675,6 +676,7 @@ return newop = self.replace_op_with_no_ovf(op) self._newoperations[-1] = newop + self._emittedoperations[newop] = None def replace_op_with_no_ovf(self, op): if op.getopnum() == rop.INT_MUL_OVF: @@ -719,6 +721,7 @@ new_descr = new_op.getdescr() new_descr.copy_all_attributes_from(old_descr) self._newoperations[old_op_pos] = new_op + self._emittedoperations[new_op] = None def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers, AbstractResOp + OpHelpers from rpython.rlib.rarithmetic import highest_bit from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper import rclass @@ -389,6 +389,8 @@ def optimize_GUARD_SUBCLASS(self, op): info = self.getptrinfo(op.getarg(0)) optimizer = self.optimizer + # must raise 'InvalidLoop' in all cases where 'info' shows the + # class cannot possibly match (see test_issue2926) if info and info.is_constant(): c = self.get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.ts.cls_of_box(c).getint() @@ -398,13 +400,29 @@ if info is not None and info.is_about_object(): known_class = info.get_known_class(optimizer.cpu) if known_class: + # Class of 'info' is exactly 'known_class'. + # We know statically if the 'guard_subclass' will pass or fail. if optimizer._check_subclass(known_class.getint(), op.getarg(1).getint()): return + else: + raise InvalidLoop( + "GUARD_SUBCLASS(known_class) proven to always fail") elif info.get_descr() is not None: - if optimizer._check_subclass(info.get_descr().get_vtable(), + # Class of 'info' is either get_descr() or a subclass of it. + # We're keeping the 'guard_subclass' at runtime only in the + # case where get_descr() is some strict parent class of + # the argument to 'guard_subclass'. + info_base_descr = info.get_descr().get_vtable() + if optimizer._check_subclass(info_base_descr, op.getarg(1).getint()): - return + return # guard_subclass always passing + elif optimizer._check_subclass(op.getarg(1).getint(), + info_base_descr): + pass # don't know, must keep the 'guard_subclass' + else: + raise InvalidLoop( + "GUARD_SUBCLASS(base_class) proven to always fail") return self.emit(op) def optimize_GUARD_NONNULL(self, op): @@ -490,11 +508,11 @@ def postprocess_GUARD_TRUE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_TRUE): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0) + self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0) self.make_constant(box, CONST_1) def optimize_GUARD_FALSE(self, op): @@ -502,11 +520,11 @@ def postprocess_GUARD_FALSE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_ZERO): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1) + self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1) self.make_constant(box, CONST_0) def optimize_ASSERT_NOT_NONE(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -74,7 +74,7 @@ print "Loop:" print '\n'.join([str(o) for o in loop.operations]) print - if expected_short: + if expected_short or getattr(info, 'short_preamble', None): print "Short Preamble:" short = info.short_preamble print '\n'.join([str(o) for o in short]) @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) @@ -9063,6 +9062,7 @@ self.optimize_loop(ops, expected) def test_same_as_preserves_info_in_the_preamble_2(self): + py.test.xfail("less efficient loop, investigate") ops = """ [i0, p0] ifoo = getfield_gc_i(p0, descr=valuedescr) @@ -9499,5 +9499,25 @@ """ self.optimize_loop(ops, expected) + def test_issue2904(self): + # we don't store advanced virtualstate information like "i1 = i2 + 1", + # which means that the following loop, when unrolled, cannot be + # optimized based on the knowledge that "i1 = i2 + 1" from the + # preamble---we can't use that knowledge. After the fix, we get + # the value "i2 + 1" passed as a third argument, possibly different + # from "i1". + ops = """ + [i1, i2] + guard_value(i1, 10) [] + i3 = int_add(i2, 1) + jump(i3, i2) + """ + expected = """ + [i1, i2, i3] + guard_value(i1, 10) [] + jump(i3, i2, i3) + """ + self.optimize_loop(ops, expected) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -10,8 +10,7 @@ from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo from rpython.jit.metainterp.optimizeopt.virtualstate import ( VirtualStateConstructor, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints @@ -22,7 +21,6 @@ if self.optunroll.short_preamble_producer is None: assert False # unreachable code op = preamble_op.op - self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, preamble_op.preamble_op, self) @@ -144,7 +142,6 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) @@ -431,8 +428,9 @@ for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): - if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): - info = self.optimizer.getinfo(arg.getarg(0)) + arg1 = self.optimizer.as_operation(arg) + if arg1 is not None and rop.is_same_as(arg1.opnum): + info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -6,8 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.optimizeopt import info from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.unroll import unrolling_iterable diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,3 +4702,112 @@ res = self.meta_interp(f, [10]) assert res == f(10) + def test_cached_info_missing(self): + py.test.skip("XXX hitting a non-translated assert in optimizeopt/heap.py, but seems not to hurt the rest") + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + height = IntVal(height) + c = IntVal(0) + h = height + total = IntVal(0) + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) + if c.intval >= iterations: + return total.intval + h = height + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2904(self): + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + def f(height, iterations): + set_param(driver, 'threshold', 4) + set_param(driver, 'trace_eagerness', 1) + c = 0 + h = height + total = 0 + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h != 0: + h = h - 1 + total = total + 1 + else: + c = c + 1 + if c >= iterations: + return total + h = height - 1 + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2926(self): + driver = JitDriver(greens = [], reds=['i', 'total', 'p']) + + class Base(object): + def do_stuff(self): + return 1000 + class Int(Base): + def __init__(self, intval): + self.intval = intval + def do_stuff(self): + return self.intval + class SubInt(Int): + pass + class Float(Base): + def __init__(self, floatval): + self.floatval = floatval + def do_stuff(self): + return int(self.floatval) + + prebuilt = [Int(i) for i in range(10)] + + @dont_look_inside + def forget_intbounds(i): + return i + + @dont_look_inside + def escape(p): + pass + + def f(i): + total = 0 + p = Base() + while True: + driver.jit_merge_point(i=i, total=total, p=p) + #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i + if i == 13: + break + total += p.do_stuff() + j = forget_intbounds(i) + if j < 10: # initial loop + p = prebuilt[i] + p.intval = j + elif j < 12: + p = Int(i) + else: + p = Float(3.14) + escape(p) + i += 1 + return total + + res = self.meta_interp(f, [0]) + assert res == f(0) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -73,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -192,6 +192,8 @@ # ____________________________________________________________ + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +376,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1644,7 +1647,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1843,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2249,7 +2253,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2493,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3000,6 +3005,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,6 +1,7 @@ import sys import time +from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rarithmetic import is_valid_int, r_longlong @@ -75,6 +76,7 @@ _stop_colors = "" @always_inline + at enforceargs(str, bool) def debug_start(category, timestamp=False): """ Start a PYPYLOG section. @@ -85,6 +87,7 @@ return _debug_start(category, timestamp) @always_inline + at enforceargs(str, bool) def debug_stop(category, timestamp=False): """ Stop a PYPYLOG section. See debug_start for docs about timestamp diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,7 +136,10 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: - with lltype.scoped_alloc(TIMEVAL) as t: + # NB: can't use lltype.scoped_malloc, because that will allocate the + # with handler in the GC, but we want to use time.time from gc.collect! + t = lltype.malloc(TIMEVAL, flavor='raw') + try: errcode = -1 if GETTIMEOFDAY_NO_TZ: errcode = c_gettimeofday(t) @@ -145,13 +148,18 @@ if rffi.cast(rffi.LONG, errcode) == 0: result = decode_timeval(t) + finally: + lltype.free(t, flavor='raw') if result != -1: return result else: # assume using ftime(3) - with lltype.scoped_alloc(TIMEB) as t: + t = lltype.malloc(TIMEB, flavor='raw') + try: c_ftime(t) result = (float(intmask(t.c_time)) + float(intmask(t.c_millitm)) * 0.001) + finally: + lltype.free(t, flavor='raw') return result return float(c_time(void)) diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -179,7 +179,7 @@ t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total0 + total == 9 def test_loop(): l = [10, 12, 15, 1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -521,11 +521,9 @@ assert path.check(file=0) def test_debug_start_stop_timestamp(self): - import sys - import time from rpython.rlib.rtimer import read_timestamp def entry_point(argv): - timestamp = int(argv[1]) + timestamp = bool(int(argv[1])) ts1 = debug_start("foo", timestamp=timestamp) ts2 = read_timestamp() ts3 = debug_stop("foo", timestamp=timestamp) From pypy.commits at gmail.com Sun Dec 9 05:22:14 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:14 -0800 (PST) Subject: [pypy-commit] pypy default: minimize difference with py3.5 Message-ID: <5c0cecd6.1c69fb81.b80a6.e595@mx.google.com> Author: Matti Picus Branch: Changeset: r95447:03cf0eb3c594 Date: 2018-12-09 11:45 +0200 http://bitbucket.org/pypy/pypy/changeset/03cf0eb3c594/ Log: minimize difference with py3.5 diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1082,6 +1082,10 @@ EINPROGRESS = cConfig.EINPROGRESS or cConfig.WSAEINPROGRESS EWOULDBLOCK = cConfig.EWOULDBLOCK or cConfig.WSAEWOULDBLOCK EAFNOSUPPORT = cConfig.EAFNOSUPPORT or cConfig.WSAEAFNOSUPPORT +# vs 2010 and above define both the constansts +WSAEINPROGRESS = cConfig.WSAEINPROGRESS or cConfig.EINPROGRESS +WSAEWOULDBLOCK = cConfig.WSAEWOULDBLOCK or cConfig.EWOULDBLOCK +WSAEAFNOSUPPORT = cConfig.WSAEAFNOSUPPORT or cConfig.EAFNOSUPPORT EISCONN = cConfig.EISCONN or cConfig.WSAEISCONN PIPE_BUF = cConfig.PIPE_BUF # may be None diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py --- a/rpython/rlib/rposix_environ.py +++ b/rpython/rlib/rposix_environ.py @@ -159,6 +159,8 @@ def envitems_llimpl(): environ = get_environ() result = [] + if not environ: + return result i = 0 while environ[i]: name_value = traits.charp2str(environ[i]) diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -707,7 +707,8 @@ address.unlock() errno = _c.geterrno() timeout = self.timeout - if timeout > 0.0 and res < 0 and errno == _c.EWOULDBLOCK: + if (timeout > 0.0 and res < 0 and + errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): tv = rffi.make(_c.timeval) rffi.setintfield(tv, 'c_tv_sec', int(timeout)) rffi.setintfield(tv, 'c_tv_usec', @@ -733,7 +734,7 @@ return (self.getsockopt_int(_c.SOL_SOCKET, _c.SO_ERROR), False) elif n == 0: - return (_c.EWOULDBLOCK, True) + return (_c.WSAEWOULDBLOCK, True) else: return (_c.geterrno(), False) From pypy.commits at gmail.com Sun Dec 9 05:22:17 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5: formatting Message-ID: <5c0cecd9.1c69fb81.ab49c.abe5@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95448:9ee32a56ba29 Date: 2018-12-09 11:46 +0200 http://bitbucket.org/pypy/pypy/changeset/9ee32a56ba29/ Log: formatting diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -707,8 +707,8 @@ address.unlock() errno = _c.geterrno() timeout = self.timeout - if (timeout > 0.0 and res < 0 and - errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): + if (timeout > 0.0 and res < 0 and + errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): tv = rffi.make(_c.timeval) rffi.setintfield(tv, 'c_tv_sec', int(timeout)) rffi.setintfield(tv, 'c_tv_usec', From pypy.commits at gmail.com Sun Dec 9 05:22:19 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Dec 2018 02:22:19 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5c0cecdb.1c69fb81.2cbef.e377@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95449:42c5ce5c6072 Date: 2018-12-09 11:49 +0200 http://bitbucket.org/pypy/pypy/changeset/42c5ce5c6072/ Log: merge py3.5 into branch diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy", allow_module_level=True) + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -58,4 +58,9 @@ .. branch: rlock-in-rpython -Backport CPython fix for `thread.RLock` \ No newline at end of file +Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). \ No newline at end of file diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -386,7 +386,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -603,8 +603,14 @@ warnoptions = pythonwarnings.split(',') + warnoptions if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in str(e.value) + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in str(excinfo.value) - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in str(e.value) +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in str(excinfo.value) - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = pytest.raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = pytest.raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = pytest.raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = pytest.raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -97,8 +97,8 @@ w_obj = space.call_function(w_inttype, w_obj) return w_obj - @unwrap_spec(nbytes=int, byteorder='text', signed=bool) - def descr_to_bytes(self, space, nbytes, byteorder, signed=False): + @unwrap_spec(length=int, byteorder='text', signed=bool) + def descr_to_bytes(self, space, length, byteorder, signed=False): """to_bytes(...) int.to_bytes(length, byteorder, *, signed=False) -> bytes @@ -121,7 +121,7 @@ """ bigint = space.bigint_w(self) try: - byte_string = bigint.tobytes(nbytes, byteorder=byteorder, + byte_string = bigint.tobytes(length, byteorder=byteorder, signed=signed) except InvalidEndiannessError: raise oefmt(space.w_ValueError, diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -378,6 +378,7 @@ assert (-8388608).to_bytes(3, 'little', signed=True) == b'\x00\x00\x80' raises(OverflowError, (-5).to_bytes, 1, 'big') raises(ValueError, (-5).to_bytes, 1, 'foo') + assert 65535 .to_bytes(length=2, byteorder='big') == b'\xff\xff' def test_negative_zero(self): x = eval("-self._long(0)") diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -44,7 +44,7 @@ if config.translation.jit_profiler == "oprofile": from rpython.jit.backend.x86 import oprofile if not oprofile.OPROFILE_AVAILABLE: - log.WARNING('oprofile support was explicitly enabled, but oprofile headers seem not to be available') + raise Exception('oprofile support was explicitly enabled, but oprofile headers seem not to be available') profile_agent = oprofile.OProfileAgent() self.with_threads = config.translation.thread diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -11,7 +11,7 @@ from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - AbstractResOp, GuardResOp + GuardResOp from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -6,7 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rlib.rarithmetic import intmask @@ -44,8 +44,9 @@ if b.has_lower and b.has_upper and b.lower == b.upper: self.make_constant_int(box, b.lower) - if isinstance(box, AbstractResOp): - dispatch_bounds_ops(self, box) + box1 = self.optimizer.as_operation(box) + if box1 is not None: + dispatch_bounds_ops(self, box1) def _optimize_guard_true_false_value(self, op): return self.emit(op) @@ -126,10 +127,11 @@ v1, v2 = v2, v1 # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): - if not self.optimizer.is_inputarg(arg1): + arg1 = self.optimizer.as_operation(arg1) + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = arg1.getarg(0) - prod_arg2 = arg1.getarg(1) + prod_arg1 = self.get_box_replacement(arg1.getarg(0)) + prod_arg2 = self.get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -279,6 +279,7 @@ self.quasi_immutable_deps = None self.replaces_guard = {} self._newoperations = [] + self._emittedoperations = {} self.optimizer = self self.optpure = None self.optheap = None @@ -292,11 +293,6 @@ self.set_optimizations(optimizations) self.setup() - def init_inparg_dict_from(self, lst): - self.inparg_dict = {} - for box in lst: - self.inparg_dict[box] = None - def set_optimizations(self, optimizations): if optimizations: self.first_optimization = optimizations[0] @@ -384,9 +380,12 @@ return info.force_box(op, optforce) return op - def is_inputarg(self, op): - return True - return op in self.inparg_dict + def as_operation(self, op): + # You should never check "isinstance(op, AbstractResOp" directly. + # Instead, use this helper. + if isinstance(op, AbstractResOp) and op in self._emittedoperations: + return op + return None def get_constant_box(self, box): box = self.get_box_replacement(box) @@ -404,6 +403,7 @@ def clear_newoperations(self): self._newoperations = [] + self._emittedoperations = {} def make_equal_to(self, op, newop): op = self.get_box_replacement(op) @@ -631,6 +631,7 @@ self._last_guard_op = None self._really_emitted_operation = op self._newoperations.append(op) + self._emittedoperations[op] = None def emit_guard_operation(self, op, pendingfields): guard_op = op # self.replace_op_with(op, op.getopnum()) @@ -675,6 +676,7 @@ return newop = self.replace_op_with_no_ovf(op) self._newoperations[-1] = newop + self._emittedoperations[newop] = None def replace_op_with_no_ovf(self, op): if op.getopnum() == rop.INT_MUL_OVF: @@ -719,6 +721,7 @@ new_descr = new_op.getdescr() new_descr.copy_all_attributes_from(old_descr) self._newoperations[old_op_pos] = new_op + self._emittedoperations[new_op] = None def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers, AbstractResOp + OpHelpers from rpython.rlib.rarithmetic import highest_bit from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper import rclass @@ -389,6 +389,8 @@ def optimize_GUARD_SUBCLASS(self, op): info = self.getptrinfo(op.getarg(0)) optimizer = self.optimizer + # must raise 'InvalidLoop' in all cases where 'info' shows the + # class cannot possibly match (see test_issue2926) if info and info.is_constant(): c = self.get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.ts.cls_of_box(c).getint() @@ -398,13 +400,29 @@ if info is not None and info.is_about_object(): known_class = info.get_known_class(optimizer.cpu) if known_class: + # Class of 'info' is exactly 'known_class'. + # We know statically if the 'guard_subclass' will pass or fail. if optimizer._check_subclass(known_class.getint(), op.getarg(1).getint()): return + else: + raise InvalidLoop( + "GUARD_SUBCLASS(known_class) proven to always fail") elif info.get_descr() is not None: - if optimizer._check_subclass(info.get_descr().get_vtable(), + # Class of 'info' is either get_descr() or a subclass of it. + # We're keeping the 'guard_subclass' at runtime only in the + # case where get_descr() is some strict parent class of + # the argument to 'guard_subclass'. + info_base_descr = info.get_descr().get_vtable() + if optimizer._check_subclass(info_base_descr, op.getarg(1).getint()): - return + return # guard_subclass always passing + elif optimizer._check_subclass(op.getarg(1).getint(), + info_base_descr): + pass # don't know, must keep the 'guard_subclass' + else: + raise InvalidLoop( + "GUARD_SUBCLASS(base_class) proven to always fail") return self.emit(op) def optimize_GUARD_NONNULL(self, op): @@ -490,11 +508,11 @@ def postprocess_GUARD_TRUE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_TRUE): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0) + self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0) self.make_constant(box, CONST_1) def optimize_GUARD_FALSE(self, op): @@ -502,11 +520,11 @@ def postprocess_GUARD_FALSE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_ZERO): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1) + self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1) self.make_constant(box, CONST_0) def optimize_ASSERT_NOT_NONE(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -74,7 +74,7 @@ print "Loop:" print '\n'.join([str(o) for o in loop.operations]) print - if expected_short: + if expected_short or getattr(info, 'short_preamble', None): print "Short Preamble:" short = info.short_preamble print '\n'.join([str(o) for o in short]) @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) @@ -9063,6 +9062,7 @@ self.optimize_loop(ops, expected) def test_same_as_preserves_info_in_the_preamble_2(self): + py.test.xfail("less efficient loop, investigate") ops = """ [i0, p0] ifoo = getfield_gc_i(p0, descr=valuedescr) @@ -9499,5 +9499,25 @@ """ self.optimize_loop(ops, expected) + def test_issue2904(self): + # we don't store advanced virtualstate information like "i1 = i2 + 1", + # which means that the following loop, when unrolled, cannot be + # optimized based on the knowledge that "i1 = i2 + 1" from the + # preamble---we can't use that knowledge. After the fix, we get + # the value "i2 + 1" passed as a third argument, possibly different + # from "i1". + ops = """ + [i1, i2] + guard_value(i1, 10) [] + i3 = int_add(i2, 1) + jump(i3, i2) + """ + expected = """ + [i1, i2, i3] + guard_value(i1, 10) [] + jump(i3, i2, i3) + """ + self.optimize_loop(ops, expected) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -10,8 +10,7 @@ from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo from rpython.jit.metainterp.optimizeopt.virtualstate import ( VirtualStateConstructor, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints @@ -22,7 +21,6 @@ if self.optunroll.short_preamble_producer is None: assert False # unreachable code op = preamble_op.op - self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, preamble_op.preamble_op, self) @@ -144,7 +142,6 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) @@ -431,8 +428,9 @@ for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): - if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): - info = self.optimizer.getinfo(arg.getarg(0)) + arg1 = self.optimizer.as_operation(arg) + if arg1 is not None and rop.is_same_as(arg1.opnum): + info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -6,8 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.optimizeopt import info from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.unroll import unrolling_iterable diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,3 +4702,112 @@ res = self.meta_interp(f, [10]) assert res == f(10) + def test_cached_info_missing(self): + py.test.skip("XXX hitting a non-translated assert in optimizeopt/heap.py, but seems not to hurt the rest") + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + height = IntVal(height) + c = IntVal(0) + h = height + total = IntVal(0) + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) + if c.intval >= iterations: + return total.intval + h = height + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2904(self): + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + def f(height, iterations): + set_param(driver, 'threshold', 4) + set_param(driver, 'trace_eagerness', 1) + c = 0 + h = height + total = 0 + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h != 0: + h = h - 1 + total = total + 1 + else: + c = c + 1 + if c >= iterations: + return total + h = height - 1 + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2926(self): + driver = JitDriver(greens = [], reds=['i', 'total', 'p']) + + class Base(object): + def do_stuff(self): + return 1000 + class Int(Base): + def __init__(self, intval): + self.intval = intval + def do_stuff(self): + return self.intval + class SubInt(Int): + pass + class Float(Base): + def __init__(self, floatval): + self.floatval = floatval + def do_stuff(self): + return int(self.floatval) + + prebuilt = [Int(i) for i in range(10)] + + @dont_look_inside + def forget_intbounds(i): + return i + + @dont_look_inside + def escape(p): + pass + + def f(i): + total = 0 + p = Base() + while True: + driver.jit_merge_point(i=i, total=total, p=p) + #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i + if i == 13: + break + total += p.do_stuff() + j = forget_intbounds(i) + if j < 10: # initial loop + p = prebuilt[i] + p.intval = j + elif j < 12: + p = Int(i) + else: + p = Float(3.14) + escape(p) + i += 1 + return total + + res = self.meta_interp(f, [0]) + assert res == f(0) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -73,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -192,6 +192,8 @@ # ____________________________________________________________ + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +376,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1644,7 +1647,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1843,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2249,7 +2253,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2493,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3000,6 +3005,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,6 +1,7 @@ import sys import time +from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rarithmetic import is_valid_int, r_longlong @@ -75,6 +76,7 @@ _stop_colors = "" @always_inline + at enforceargs(str, bool) def debug_start(category, timestamp=False): """ Start a PYPYLOG section. @@ -85,6 +87,7 @@ return _debug_start(category, timestamp) @always_inline + at enforceargs(str, bool) def debug_stop(category, timestamp=False): """ Stop a PYPYLOG section. See debug_start for docs about timestamp diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -708,7 +708,7 @@ errno = _c.geterrno() timeout = self.timeout if (timeout > 0.0 and res < 0 and - errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): + errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): tv = rffi.make(_c.timeval) rffi.setintfield(tv, 'c_tv_sec', int(timeout)) rffi.setintfield(tv, 'c_tv_usec', diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,7 +136,10 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: - with lltype.scoped_alloc(TIMEVAL) as t: + # NB: can't use lltype.scoped_malloc, because that will allocate the + # with handler in the GC, but we want to use time.time from gc.collect! + t = lltype.malloc(TIMEVAL, flavor='raw') + try: errcode = -1 if GETTIMEOFDAY_NO_TZ: errcode = c_gettimeofday(t) @@ -145,13 +148,18 @@ if rffi.cast(rffi.LONG, errcode) == 0: result = decode_timeval(t) + finally: + lltype.free(t, flavor='raw') if result != -1: return result else: # assume using ftime(3) - with lltype.scoped_alloc(TIMEB) as t: + t = lltype.malloc(TIMEB, flavor='raw') + try: c_ftime(t) result = (float(intmask(t.c_time)) + float(intmask(t.c_millitm)) * 0.001) + finally: + lltype.free(t, flavor='raw') return result return float(c_time(void)) diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -179,7 +179,7 @@ t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total0 + total == 9 def test_loop(): l = [10, 12, 15, 1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -521,11 +521,9 @@ assert path.check(file=0) def test_debug_start_stop_timestamp(self): - import sys - import time from rpython.rlib.rtimer import read_timestamp def entry_point(argv): - timestamp = int(argv[1]) + timestamp = bool(int(argv[1])) ts1 = debug_start("foo", timestamp=timestamp) ts2 = read_timestamp() ts3 = debug_stop("foo", timestamp=timestamp) From pypy.commits at gmail.com Mon Dec 10 17:55:36 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 10 Dec 2018 14:55:36 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: update info Message-ID: <5c0eeee8.1c69fb81.38f8f.cf5f@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5920:8969487c4ef9 Date: 2018-12-11 00:55 +0200 http://bitbucket.org/pypy/extradoc/changeset/8969487c4ef9/ Log: update info diff --git a/sprintinfo/ddorf2019/announce.txt b/sprintinfo/ddorf2019/announce.txt --- a/sprintinfo/ddorf2019/announce.txt +++ b/sprintinfo/ddorf2019/announce.txt @@ -12,6 +12,14 @@ - improve Python 3.6 support - discuss benchmarking situation - progress on utf-8 branches +- cpyext +- packaging: are we ready to upload to PyPI? + - [issue 2617](https://bitbucket.org/pypy/pypy/issues/2617) - we expose too many functions from lib-pypy.so + - mutlilinux2010 + - formulate an ABI name and upgrade policy +- memoryview(ctypes.Structure) + + Location -------- diff --git a/sprintinfo/ddorf2019/people.txt b/sprintinfo/ddorf2019/people.txt --- a/sprintinfo/ddorf2019/people.txt +++ b/sprintinfo/ddorf2019/people.txt @@ -1,5 +1,5 @@ -People coming to the Duesseldorf sprint October 2010 -==================================================== +People coming to the Duesseldorf sprint Feb 4-9 2019 +===================================================== People who have a ``?`` in their arrive/depart or accomodation column are known to be coming but there are no details @@ -9,6 +9,7 @@ Name Arrive/Depart Accomodation ============================ ============== ===================== Carl Friedrich Bolz-Tereick always there private +Matti Picus Feb 4? - 9? any suggestions?? ============================ ============== ===================== From pypy.commits at gmail.com Tue Dec 11 08:01:06 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 11 Dec 2018 05:01:06 -0800 (PST) Subject: [pypy-commit] pypy default: add more ideas, cleanup duplicate TOC references to architecture Message-ID: <5c0fb512.1c69fb81.cf134.cb28@mx.google.com> Author: Matti Picus Branch: Changeset: r95450:00010b268b1f Date: 2018-12-11 14:57 +0200 http://bitbucket.org/pypy/pypy/changeset/00010b268b1f/ Log: add more ideas, cleanup duplicate TOC references to architecture diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -4,7 +4,7 @@ .. contents:: This document gives an overview of the goals and architecture of PyPy. If you're -interested in :ref:`using PyPy ` or :ref:`hacking on it `, +interested in :ref:`using PyPy ` or hacking on it, have a look at our :ref:`getting started ` section. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -29,6 +29,7 @@ :maxdepth: 1 introduction + architecture install build windows @@ -59,7 +60,6 @@ :maxdepth: 2 contributing - architecture configuration project-ideas project-documentation diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst --- a/pypy/doc/project-documentation.rst +++ b/pypy/doc/project-documentation.rst @@ -28,7 +28,6 @@ .. toctree:: :hidden: - architecture coding-guide sprint-reports extradoc diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -111,9 +111,12 @@ -------------- Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. -Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy -test suite. We could use help analyzing the failures and fixing them either -as patches to upstream NumPy, or as fixes to PyPy. +Release PyPy2.7-v6.0 still fails about 10 of the ~6000 test in the NumPy +test suite. We need to improve our ctypes structure -> memoryview conversions_, +and to refactor the way `NumPy adds docstrings`_. + +.. _conversions: https://bitbucket.org/pypy/pypy/issues/2930 +.. _`NumPy adds docstrings`: https://github.com/numpy/numpy/issues/10167 We also are looking for help in how to hijack NumPy dtype conversion and ufunc calls to allow the JIT to make them fast, using our internal _numpypy @@ -165,8 +168,33 @@ Or maybe not. We can also play around with the idea of using a single representation: as a byte string in utf-8. (This idea needs some extra logic -for efficient indexing, like a cache.) +for efficient indexing, like a cache.) Work has begun on the ``unicode-utf`` +and ``unicode-utf8-py3`` branches. More is needed, for instance there are +SIMD optimizations that are not yet used. +Convert RPython to Python3 +-------------------------- + +The world is moving on, we should too. + +Improve performance +------------------- + +* Make uninlined Python-level calls faster +* Switch to a `sea-of-nodes`_ IR, or a `Lua-Jit`_-like IR which iterates on + on the sea-of-nodes approach +* Use real register-allocation +* Improve instruction selection / scheduling +* Create a hybrid tracing/method JIT + +.. _`sea-of-nodes`: https://darksi.de/d.sea-of-nodes/ +.. _`Lua-JIT`: http://wiki.luajit.org/SSA-IR-2.0 + +Improve warmup +-------------- +* Interpreter speed-ups +* Optimize while tracing +* Cache information between runs Translation Toolchain --------------------- @@ -234,6 +262,27 @@ .. _runner: http://speed.pypy.org .. _`CPython site`: https://speed.python.org/ + +Interfacing with C +------------------ + +While we could make ``cpyext`` faster_, we would also like to explore other +ideas. It seems cffi is only appropriate for small to medium-sized extensions, +and it is hard to imagine NumPy abandoning the C-API. Here are a few ideas: +* Extend Cython to have a backend that can be understood by the JIT +* Collaborate with C-extension authors to ensure full PyPy support (see below) +* Put PyPy compatible packages on PyPI and in conda + + +.. _faster: https://morepypy.blogspot.com/2018/09#next-steps + +Support more platforms +---------------------- + +We have a plan for a `Windows 64`_ port. + +.. _`Windows 64`: windows.html#what-is-missing-for-a-full-64-bit-translation + ====================================== Make more python modules pypy-friendly ====================================== From pypy.commits at gmail.com Tue Dec 11 15:47:33 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 11 Dec 2018 12:47:33 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Clean up test using pytest's builtin fixtures Message-ID: <5c102265.1c69fb81.75f1d.b51e@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95451:b73f30700869 Date: 2018-12-11 20:26 +0000 http://bitbucket.org/pypy/pypy/changeset/b73f30700869/ Log: Clean up test using pytest's builtin fixtures diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py @@ -1,50 +1,35 @@ # derived from test_random_things.py +import pytest + from ctypes import * -import sys -class TestCallbackTraceback: - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. +_rawffi = pytest.importorskip('_rawffi') + +# +# This test makes sure the exception types *and* the exception +# value is printed correctly. + + at pytest.mark.skipif("sys.flags.inspect") +def test_SystemExit(monkeypatch, capsys): + """ + When an exception is raised in a ctypes callback function, the C + code prints a traceback. When SystemExit is raised, the interpreter + normally exits immediately. + """ + def callback_func(arg): + raise SystemExit(42) + def custom_exit(value): + raise Exception("<<>>" % (value,)) + monkeypatch.setattr(_rawffi, 'exit', custom_exit) + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) + out, err = capsys.readouterr() + assert not err + cb2(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead a additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import StringIO - old_stderr = sys.stderr - logger = sys.stderr = StringIO.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() - - def test_SystemExit(self): - import _rawffi - if sys.flags.inspect: - skip("requires sys.flags.inspect == 0") - def callback_func(arg): - raise SystemExit(42) - def custom_exit(value): - raise Exception("<<>>" % (value,)) - original_exit = _rawffi.exit - try: - _rawffi.exit = custom_exit - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) - out = self.capture_stderr(cb2, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - finally: - _rawffi.exit = original_exit + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" From pypy.commits at gmail.com Tue Dec 11 15:47:35 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 11 Dec 2018 12:47:35 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Convert some test methods to functions Message-ID: <5c102267.1c69fb81.74c97.db01@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95452:007da6e72200 Date: 2018-12-11 20:46 +0000 http://bitbucket.org/pypy/pypy/changeset/007da6e72200/ Log: Convert some test methods to functions diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py @@ -1,56 +1,54 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestAnon(BaseCTypesTestChecker): - def test_nested(self): - class ANON_S(Structure): - _fields_ = [("a", c_int)] +def test_nested(): + class ANON_S(Structure): + _fields_ = [("a", c_int)] - class ANON_U(Union): - _fields_ = [("_", ANON_S), - ("b", c_int)] - _anonymous_ = ["_"] + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON_U), - ("y", c_int)] - _anonymous_ = ["_"] + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] - assert Y.x.offset == 0 - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) - assert Y._.offset == sizeof(c_int) - assert Y.y.offset == sizeof(c_int) * 2 + assert Y.x.offset == 0 + assert Y.a.offset == sizeof(c_int) + assert Y.b.offset == sizeof(c_int) + assert Y._.offset == sizeof(c_int) + assert Y.y.offset == sizeof(c_int) * 2 - assert Y._names_ == ['x', 'a', 'b', 'y'] + assert Y._names_ == ['x', 'a', 'b', 'y'] - def test_anonymous_fields_on_instance(self): - # this is about the *instance-level* access of anonymous fields, - # which you'd guess is the most common, but used not to work - # (issue #2230) +def test_anonymous_fields_on_instance(): + # this is about the *instance-level* access of anonymous fields, + # which you'd guess is the most common, but used not to work + # (issue #2230) - class B(Structure): - _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] - class A(Structure): - _anonymous_ = ["b"] - _fields_ = [("b", B)] + class B(Structure): + _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] + class A(Structure): + _anonymous_ = ["b"] + _fields_ = [("b", B)] - a = A() - a.x = 5 - assert a.x == 5 - assert a.b.x == 5 - a.b.x += 1 - assert a.x == 6 + a = A() + a.x = 5 + assert a.x == 5 + assert a.b.x == 5 + a.b.x += 1 + assert a.x == 6 - class C(Structure): - _anonymous_ = ["a"] - _fields_ = [("v", c_int), ("a", A)] + class C(Structure): + _anonymous_ = ["a"] + _fields_ = [("v", c_int), ("a", A)] - c = C() - c.v = 3 - c.y = -8 - assert c.v == 3 - assert c.y == c.a.y == c.a.b.y == -8 - assert not hasattr(c, 'b') + c = C() + c.v = 3 + c.y = -8 + assert c.v == 3 + assert c.y == c.a.y == c.a.b.y == -8 + assert not hasattr(c, 'b') diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py @@ -1,67 +1,64 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestArray(BaseCTypesTestChecker): - def test_slice(self): - values = range(5) - numarray = c_int * 5 +def test_slice(): + values = range(5) + numarray = c_int * 5 - na = numarray(*(c_int(x) for x in values)) + na = numarray(*(c_int(x) for x in values)) - assert list(na[0:0]) == [] - assert list(na[:]) == values - assert list(na[:10]) == values + assert list(na[0:0]) == [] + assert list(na[:]) == values + assert list(na[:10]) == values - def test_init_again(self): - sz = (c_char * 3)() - addr1 = addressof(sz) - sz.__init__(*"foo") - addr2 = addressof(sz) - assert addr1 == addr2 +def test_init_again(): + sz = (c_char * 3)() + addr1 = addressof(sz) + sz.__init__(*"foo") + addr2 = addressof(sz) + assert addr1 == addr2 -class TestSophisticatedThings(BaseCTypesTestChecker): - def test_array_of_structures(self): - class X(Structure): - _fields_ = [('x', c_int), ('y', c_int)] +def test_array_of_structures(): + class X(Structure): + _fields_ = [('x', c_int), ('y', c_int)] - Y = X * 2 - y = Y() - x = X() - x.y = 3 - y[1] = x - assert y[1].y == 3 + Y = X * 2 + y = Y() + x = X() + x.y = 3 + y[1] = x + assert y[1].y == 3 - def test_output_simple(self): - A = c_char * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' +def test_output_simple(): + A = c_char * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != '' - A = c_wchar * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = c_wchar * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != '' - def test_output_simple_array(self): - A = c_char * 10 - AA = A * 10 - aa = AA() - assert aa[0] != '' +def test_output_simple_array(): + A = c_char * 10 + AA = A * 10 + aa = AA() + assert aa[0] != '' - def test_output_complex_test(self): - class Car(Structure): - _fields_ = [("brand", c_char * 10), - ("speed", c_float), - ("owner", c_char * 10)] +def test_output_complex_test(): + class Car(Structure): + _fields_ = [("brand", c_char * 10), + ("speed", c_float), + ("owner", c_char * 10)] - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" - with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") + assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) + assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" + assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" + with pytest.raises(ValueError): + Car("abcdefghiop", 42.0, "12345") - A = Car._fields_[2][1] - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = Car._fields_[2][1] + TP = POINTER(A) + x = TP(A()) + assert x[0] != '' diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/pypy/module/test_lib_pypy/ctypes_tests/test_base.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_base.py @@ -1,26 +1,23 @@ -from .support import WhiteBoxTests - from ctypes import * # WhiteBoxTests -class TestCTypesBase(WhiteBoxTests): - def test_pointer(self): - p = pointer(pointer(c_int(2))) - x = p[0] - assert x._base is p +def test_pointer(): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p - def test_structure(self): - class X(Structure): - _fields_ = [('x', POINTER(c_int)), - ('y', POINTER(c_int))] +def test_structure(): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] - x = X() - assert x.y._base is x - assert x.y._index == 1 + x = X() + assert x.y._base is x + assert x.y._index == 1 - def test_array(self): - X = POINTER(c_int) * 24 - x = X() - assert x[16]._base is x - assert x[16]._index == 16 +def test_array(): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py @@ -2,21 +2,18 @@ from ctypes import * -class TestBitField: - def test_set_fields_attr(self): - class A(Structure): - pass - A._fields_ = [("a", c_byte), - ("b", c_ubyte)] +def test_set_fields_attr(): + class A(Structure): + pass + A._fields_ = [("a", c_byte), ("b", c_ubyte)] - def test_set_fields_attr_bitfields(self): - class A(Structure): - pass - A._fields_ = [("a", POINTER(A)), - ("b", c_ubyte, 4)] +def test_set_fields_attr_bitfields(): + class A(Structure): + pass + A._fields_ = [("a", POINTER(A)), ("b", c_ubyte, 4)] - def test_set_fields_cycle_fails(self): - class A(Structure): - pass - with pytest.raises(AttributeError): - A._fields_ = [("a", A)] +def test_set_fields_cycle_fails(): + class A(Structure): + pass + with pytest.raises(AttributeError): + A._fields_ = [("a", A)] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py @@ -1,41 +1,38 @@ from ctypes import * -from .support import BaseCTypesTestChecker -class TestStringBuffer(BaseCTypesTestChecker): +def test_buffer(): + b = create_string_buffer(32) + assert len(b) == 32 + assert sizeof(b) == 32 * sizeof(c_char) + assert type(b[0]) is str - def test_buffer(self): - b = create_string_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(33L) + assert len(b) == 33 + assert sizeof(b) == 33 * sizeof(c_char) + assert type(b[0]) is str - b = create_string_buffer(33L) - assert len(b) == 33 - assert sizeof(b) == 33 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer("abc") + assert len(b) == 4 # trailing nul char + assert sizeof(b) == 4 * sizeof(c_char) + assert type(b[0]) is str + assert b[0] == "a" + assert b[:] == "abc\0" - b = create_string_buffer("abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" +def test_from_buffer(): + b1 = bytearray("abcde") + b = (c_char * 5).from_buffer(b1) + assert b[2] == "c" + # + b1 = bytearray("abcd") + b = c_int.from_buffer(b1) + assert b.value in (1684234849, # little endian + 1633837924) # big endian - def test_from_buffer(self): - b1 = bytearray("abcde") - b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" - # - b1 = bytearray("abcd") - b = c_int.from_buffer(b1) - assert b.value in (1684234849, # little endian - 1633837924) # big endian - - def test_from_buffer_keepalive(self): - # Issue #2878 - b1 = bytearray("ab") - array = (c_uint16 * 32)() - array[6] = c_uint16.from_buffer(b1) - # this is also what we get on CPython. I don't think it makes - # sense because the array contains just a copy of the number. - assert array._objects == {'6': b1} +def test_from_buffer_keepalive(): + # Issue #2878 + b1 = bytearray("ab") + array = (c_uint16 * 32)() + array[6] = c_uint16.from_buffer(b1) + # this is also what we get on CPython. I don't think it makes + # sense because the array contains just a copy of the number. + assert array._objects == {'6': b1} diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py @@ -5,10 +5,6 @@ class TestCallbacks(BaseCTypesTestChecker): functype = CFUNCTYPE -## def tearDown(self): -## import gc -## gc.collect() - def callback(self, *args): self.got_args = args return args[-1] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py @@ -1,31 +1,30 @@ +import pytest + from ctypes import * -import sys, py -from .support import BaseCTypesTestChecker -class TestCast(BaseCTypesTestChecker): +def test_cast_functype(dll): + # make sure we can cast function type + my_sqrt = dll.my_sqrt + saved_objects = my_sqrt._objects.copy() + sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) + assert sqrt(4.0) == 2.0 + assert not cast(0, CFUNCTYPE(c_int)) + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) - def test_cast_functype(self, dll): - # make sure we can cast function type - my_sqrt = dll.my_sqrt - saved_objects = my_sqrt._objects.copy() - sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) - assert sqrt(4.0) == 2.0 - assert not cast(0, CFUNCTYPE(c_int)) - # - assert sqrt._objects is my_sqrt._objects # on CPython too - my_sqrt._objects.clear() - my_sqrt._objects.update(saved_objects) +def test_cast_argumenterror(): + param = c_uint(42) + with pytest.raises(ArgumentError): + cast(param, c_void_p) - def test_cast_argumenterror(self): - param = c_uint(42) - py.test.raises(ArgumentError, "cast(param, c_void_p)") - - def test_c_bool(self): - x = c_bool(42) - assert x.value is True - x = c_bool(0.0) - assert x.value is False - x = c_bool("") - assert x.value is False - x = c_bool(['yadda']) - assert x.value is True +def test_c_bool(): + x = c_bool(42) + assert x.value is True + x = c_bool(0.0) + assert x.value is False + x = c_bool("") + assert x.value is False + x = c_bool(['yadda']) + assert x.value is True diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py @@ -3,19 +3,17 @@ import ctypes _rawffi = pytest.importorskip('_rawffi') # PyPy-only -class TestErrno: +def test_errno_saved_and_restored(): + def check(): + assert _rawffi.get_errno() == 42 + assert ctypes.get_errno() == old + check.free_temp_buffers = lambda *args: None + f = ctypes._CFuncPtr() + old = _rawffi.get_errno() + f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO + ctypes.set_errno(42) + f._call_funcptr(check) + assert _rawffi.get_errno() == old + ctypes.set_errno(0) - def test_errno_saved_and_restored(self): - def check(): - assert _rawffi.get_errno() == 42 - assert ctypes.get_errno() == old - check.free_temp_buffers = lambda *args: None - f = ctypes._CFuncPtr() - old = _rawffi.get_errno() - f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO - ctypes.set_errno(42) - f._call_funcptr(check) - assert _rawffi.get_errno() == old - ctypes.set_errno(0) - - # see also test_functions.test_errno +# see also test_functions.test_errno diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py @@ -5,241 +5,239 @@ import py from ctypes import * -from .support import BaseCTypesTestChecker -class TestExtra(BaseCTypesTestChecker): - def test_primitive_pointer(self): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 +def test_primitive_pointer(): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 - ## *** the rest is commented out because it should work but occasionally - ## *** trigger a ctypes bug (SourceForge bug #1467852). *** - ## q = pointer(y) - ## pp[0] = q # pp ---> p ---> y = 14 - ## assert y.value == 14 # (^^^ not q! ) - ## assert p.contents.value == 14 - ## assert pp.contents.contents.value == 14 - ## q.contents = x - ## assert pp.contents.contents.value == 14 +## *** the rest is commented out because it should work but occasionally +## *** trigger a ctypes bug (SourceForge bug #1467852). *** +## q = pointer(y) +## pp[0] = q # pp ---> p ---> y = 14 +## assert y.value == 14 # (^^^ not q! ) +## assert p.contents.value == 14 +## assert pp.contents.contents.value == 14 +## q.contents = x +## assert pp.contents.contents.value == 14 - def test_char_p(self): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" +def test_char_p(): + x = c_char_p("hello\x00world") + assert x.value == "hello" + x.value = "world" + assert x.value == "world" - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + p = pointer(x) + assert p[0] == x.value == "world" + p[0] = "other" + assert x.value == p.contents.value == p[0] == "other" - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + myarray = (c_char_p * 10)() + myarray[7] = "hello" + assert isinstance(myarray[7], str) + assert myarray[7] == "hello" - def test_struct(self): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] +def test_struct(): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 - def test_ptr_array(self): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) +def test_ptr_array(): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 - def test_void_p(self): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 +def test_void_p(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 - def test_char_array(self): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' +def test_char_array(): + a = (c_char * 3)() + a[0] = 'x' + a[1] = 'y' + assert a.value == 'xy' + a[2] = 'z' + assert a.value == 'xyz' - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = "nxw" + assert b[0] == 'n' + assert b[1] == 'x' + assert b[2] == 'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = "?" + assert b[0] == '?' + assert b[1] == '\x00' + assert b[2] == 'w' - class S(Structure): - _fields_ = [('p', POINTER(c_char))] + class S(Structure): + _fields_ = [('p', POINTER(c_char))] - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s = S() + s.p = b + s.p.contents.value = '!' + assert b.value == '!' - assert len(create_string_buffer(0)) == 0 + assert len(create_string_buffer(0)) == 0 - def test_array(self): - a = (c_int * 10)() +def test_array(): + a = (c_int * 10)() - class S(Structure): - _fields_ = [('p', POINTER(c_int))] + class S(Structure): + _fields_ = [('p', POINTER(c_int))] - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] - def test_truth_value(self): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... +def test_truth_value(): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) - assert not c_double(0.0) - assert not c_ulonglong(0) - assert c_ulonglong(2L**42) + assert c_int(12) + assert not c_int(0) # a bit strange, if you ask me + assert c_int(-1) + assert not c_byte(0) + assert not c_char('\x00') # hum + assert not c_float(0.0) + assert not c_double(0.0) + assert not c_ulonglong(0) + assert c_ulonglong(2L**42) - assert c_char_p("hello") - assert c_char_p("") - assert not c_char_p(None) + assert c_char_p("hello") + assert c_char_p("") + assert not c_char_p(None) - assert not c_void_p() + assert not c_void_p() - def test_sizeof(self): - x = create_string_buffer(117) - assert sizeof(x) == 117 # assumes that chars are one byte each - x = (c_int * 42)() - assert sizeof(x) == 42 * sizeof(c_int) +def test_sizeof(): + x = create_string_buffer(117) + assert sizeof(x) == 117 # assumes that chars are one byte each + x = (c_int * 42)() + assert sizeof(x) == 42 * sizeof(c_int) - def test_convert_pointers(self, dll): - func = dll._testfunc_p_p - func.restype = c_char_p +def test_convert_pointers(dll): + func = dll._testfunc_p_p + func.restype = c_char_p - # automatic conversions to c_char_p - func.argtypes = [c_char_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func(create_string_buffer("hello")) == "hello" + # automatic conversions to c_char_p + func.argtypes = [c_char_p] + assert func("hello") == "hello" + assert func(c_char_p("hello")) == "hello" + assert func((c_char * 6)(*"hello")) == "hello" + assert func(create_string_buffer("hello")) == "hello" - # automatic conversions to c_void_p - func.argtypes = [c_void_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func((c_byte * 6)(104,101,108,108,111)) =="hello" - assert func(create_string_buffer("hello")) == "hello" + # automatic conversions to c_void_p + func.argtypes = [c_void_p] + assert func("hello") == "hello" + assert func(c_char_p("hello")) == "hello" + assert func((c_char * 6)(*"hello")) == "hello" + assert func((c_byte * 6)(104,101,108,108,111)) =="hello" + assert func(create_string_buffer("hello")) == "hello" - def test_varsize_cast(self): - import struct - N = struct.calcsize("l") - x = c_long() - p = cast(pointer(x), POINTER(c_ubyte*N)) - for i, c in enumerate(struct.pack("l", 12345678)): - p.contents[i] = ord(c) - assert x.value == 12345678 +def test_varsize_cast(): + import struct + N = struct.calcsize("l") + x = c_long() + p = cast(pointer(x), POINTER(c_ubyte*N)) + for i, c in enumerate(struct.pack("l", 12345678)): + p.contents[i] = ord(c) + assert x.value == 12345678 - def test_cfunctype_inspection(self): - T = CFUNCTYPE(c_int, c_ubyte) - # T.argtypes and T.restype don't work, must use a dummy instance - assert list(T().argtypes) == [c_ubyte] - assert T().restype == c_int +def test_cfunctype_inspection(): + T = CFUNCTYPE(c_int, c_ubyte) + # T.argtypes and T.restype don't work, must use a dummy instance + assert list(T().argtypes) == [c_ubyte] + assert T().restype == c_int - def test_from_param(self): - # other working cases of from_param - assert isinstance(c_void_p.from_param((c_int * 4)()), c_int*4) +def test_from_param(): + # other working cases of from_param + assert isinstance(c_void_p.from_param((c_int * 4)()), c_int * 4) - def test_array_mul(self): - assert c_int * 10 == 10 * c_int == c_int * 10L == 10L * c_int - py.test.raises(TypeError, 'c_int * c_int') - py.test.raises(TypeError, 'c_int * (-1.5)') - py.test.raises(TypeError, 'c_int * "foo"') - py.test.raises(TypeError, '(-1.5) * c_int') - py.test.raises(TypeError, '"foo" * c_int') +def test_array_mul(): + assert c_int * 10 == 10 * c_int == c_int * 10L == 10L * c_int + py.test.raises(TypeError, 'c_int * c_int') + py.test.raises(TypeError, 'c_int * (-1.5)') + py.test.raises(TypeError, 'c_int * "foo"') + py.test.raises(TypeError, '(-1.5) * c_int') + py.test.raises(TypeError, '"foo" * c_int') - def test_cast_none(self): - def check(P): - x = cast(None, P) - assert isinstance(x, P) - assert not x - check(c_void_p) - check(c_char_p) - check(POINTER(c_int)) - check(POINTER(c_int * 10)) +def test_cast_none(): + def check(P): + x = cast(None, P) + assert isinstance(x, P) + assert not x + check(c_void_p) + check(c_char_p) + check(POINTER(c_int)) + check(POINTER(c_int * 10)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py b/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py @@ -1,6 +1,5 @@ from ctypes import * -class TestCFuncPtr: - def test_restype(self, dll): - foo = dll.my_unused_function - assert foo.restype is c_int # by default +def test_restype(dll): + foo = dll.my_unused_function + assert foo.restype is c_int # by default diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py @@ -7,238 +7,236 @@ return CDLL(str(sofile), use_errno=True) -class TestFunctions(object): +def test_char_result(dll): + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_char + result = f(0, 0, 0, 0, 0, 0) + assert result == '\x00' - def test_char_result(self, dll): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_char - result = f(0, 0, 0, 0, 0, 0) - assert result == '\x00' +def test_boolresult(dll): + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_bool + false_result = f(0, 0, 0, 0, 0, 0) + assert false_result is False + true_result = f(1, 0, 0, 0, 0, 0) + assert true_result is True - def test_boolresult(self, dll): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_bool - false_result = f(0, 0, 0, 0, 0, 0) - assert false_result is False - true_result = f(1, 0, 0, 0, 0, 0) - assert true_result is True +def test_unicode_function_name(dll): + f = dll[u'_testfunc_i_bhilfd'] + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_int + result = f(1, 2, 3, 4, 5.0, 6.0) + assert result == 21 - def test_unicode_function_name(self, dll): - f = dll[u'_testfunc_i_bhilfd'] - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_int - result = f(1, 2, 3, 4, 5.0, 6.0) - assert result == 21 +def test_truncate_python_longs(dll): + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_int + x = sys.maxint * 2 + result = f(x, x, x, x, 0, 0) + assert result == -8 - def test_truncate_python_longs(self, dll): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_int - x = sys.maxint * 2 - result = f(x, x, x, x, 0, 0) - assert result == -8 +def test_convert_pointers(dll): + f = dll.deref_LP_c_char_p + f.restype = c_char + f.argtypes = [POINTER(c_char_p)] + # + s = c_char_p('hello world') + ps = pointer(s) + assert f(ps) == 'h' + assert f(s) == 'h' # automatic conversion from char** to char* - def test_convert_pointers(self, dll): - f = dll.deref_LP_c_char_p - f.restype = c_char - f.argtypes = [POINTER(c_char_p)] - # - s = c_char_p('hello world') - ps = pointer(s) - assert f(ps) == 'h' - assert f(s) == 'h' # automatic conversion from char** to char* +################################################################ - ################################################################ +def test_call_some_args(dll): + f = dll.my_strchr + f.argtypes = [c_char_p] + f.restype = c_char_p + result = f("abcd", ord("b")) + assert result == "bcd" - def test_call_some_args(self, dll): - f = dll.my_strchr - f.argtypes = [c_char_p] - f.restype = c_char_p - result = f("abcd", ord("b")) - assert result == "bcd" +def test_keepalive_buffers(monkeypatch, dll): + import gc + f = dll.my_strchr + f.argtypes = [c_char_p] + f.restype = c_char_p + # + orig__call_funcptr = f._call_funcptr + def _call_funcptr(funcptr, *newargs): + gc.collect() + gc.collect() + gc.collect() + return orig__call_funcptr(funcptr, *newargs) + monkeypatch.setattr(f, '_call_funcptr', _call_funcptr) + # + result = f("abcd", ord("b")) + assert result == "bcd" - def test_keepalive_buffers(self, monkeypatch, dll): - import gc - f = dll.my_strchr - f.argtypes = [c_char_p] - f.restype = c_char_p - # - orig__call_funcptr = f._call_funcptr - def _call_funcptr(funcptr, *newargs): - gc.collect() - gc.collect() - gc.collect() - return orig__call_funcptr(funcptr, *newargs) - monkeypatch.setattr(f, '_call_funcptr', _call_funcptr) - # - result = f("abcd", ord("b")) - assert result == "bcd" +def test_caching_bug_1(dll): + # the same test as test_call_some_args, with two extra lines + # in the middle that trigger caching in f._ptr, which then + # makes the last two lines fail + f = dll.my_strchr + f.argtypes = [c_char_p, c_int] + f.restype = c_char_p + result = f("abcd", ord("b")) + assert result == "bcd" + result = f("abcd", ord("b"), 42) + assert result == "bcd" - def test_caching_bug_1(self, dll): - # the same test as test_call_some_args, with two extra lines - # in the middle that trigger caching in f._ptr, which then - # makes the last two lines fail - f = dll.my_strchr - f.argtypes = [c_char_p, c_int] - f.restype = c_char_p - result = f("abcd", ord("b")) - assert result == "bcd" - result = f("abcd", ord("b"), 42) - assert result == "bcd" +def test_argument_conversion_and_checks(dll): + #This test is designed to check for segfaults if the wrong type of argument is passed as parameter + strlen = dll.my_strchr + strlen.argtypes = [c_char_p, c_int] + strlen.restype = c_char_p + assert strlen("eggs", ord("g")) == "ggs" - def test_argument_conversion_and_checks(self, dll): - #This test is designed to check for segfaults if the wrong type of argument is passed as parameter - strlen = dll.my_strchr - strlen.argtypes = [c_char_p, c_int] - strlen.restype = c_char_p - assert strlen("eggs", ord("g")) == "ggs" + # Should raise ArgumentError, not segfault + with pytest.raises(ArgumentError): + strlen(0, 0) + with pytest.raises(ArgumentError): + strlen(False, 0) - # Should raise ArgumentError, not segfault - with pytest.raises(ArgumentError): - strlen(0, 0) - with pytest.raises(ArgumentError): - strlen(False, 0) +def test_union_as_passed_value(dll): + class UN(Union): + _fields_ = [("x", c_short), + ("y", c_long)] + dll.ret_un_func.restype = UN + dll.ret_un_func.argtypes = [UN] + A = UN * 2 + a = A() + a[1].x = 33 + u = dll.ret_un_func(a[1]) + assert u.y == 33 * 10000 - def test_union_as_passed_value(self, dll): - class UN(Union): - _fields_ = [("x", c_short), - ("y", c_long)] - dll.ret_un_func.restype = UN - dll.ret_un_func.argtypes = [UN] - A = UN * 2 - a = A() - a[1].x = 33 - u = dll.ret_un_func(a[1]) - assert u.y == 33 * 10000 +def test_cache_funcptr(dll): + tf_b = dll.tf_b + tf_b.restype = c_byte + tf_b.argtypes = (c_byte,) + assert tf_b(-126) == -42 + ptr = tf_b._ptr + assert ptr is not None + assert tf_b(-126) == -42 + assert tf_b._ptr is ptr - def test_cache_funcptr(self, dll): - tf_b = dll.tf_b - tf_b.restype = c_byte - tf_b.argtypes = (c_byte,) - assert tf_b(-126) == -42 - ptr = tf_b._ptr - assert ptr is not None - assert tf_b(-126) == -42 - assert tf_b._ptr is ptr +def test_custom_from_param(dll): + class A(c_byte): + @classmethod + def from_param(cls, obj): + seen.append(obj) + return -126 + tf_b = dll.tf_b + tf_b.restype = c_byte + tf_b.argtypes = (c_byte,) + tf_b.argtypes = [A] + seen = [] + assert tf_b("yadda") == -42 + assert seen == ["yadda"] - def test_custom_from_param(self, dll): - class A(c_byte): - @classmethod - def from_param(cls, obj): - seen.append(obj) - return -126 - tf_b = dll.tf_b - tf_b.restype = c_byte - tf_b.argtypes = (c_byte,) - tf_b.argtypes = [A] - seen = [] - assert tf_b("yadda") == -42 - assert seen == ["yadda"] + at pytest.mark.xfail(reason="warnings are disabled") +def test_warnings(dll): + import warnings + warnings.simplefilter("always") + with warnings.catch_warnings(record=True) as w: + dll.get_an_integer() + assert len(w) == 1 + assert issubclass(w[0].category, RuntimeWarning) + assert "C function without declared arguments called" in str(w[0].message) - @pytest.mark.xfail(reason="warnings are disabled") - def test_warnings(self, dll): - import warnings - warnings.simplefilter("always") - with warnings.catch_warnings(record=True) as w: - dll.get_an_integer() - assert len(w) == 1 - assert issubclass(w[0].category, RuntimeWarning) - assert "C function without declared arguments called" in str(w[0].message) + at pytest.mark.xfail +def test_errcheck(dll): + import warnings + def errcheck(result, func, args): + assert result == -42 + assert type(result) is int + arg, = args + assert arg == -126 + assert type(arg) is int + return result + # + tf_b = dll.tf_b + tf_b.restype = c_byte + tf_b.argtypes = (c_byte,) + tf_b.errcheck = errcheck + assert tf_b(-126) == -42 + del tf_b.errcheck + with warnings.catch_warnings(record=True) as w: + dll.get_an_integer.argtypes = [] + dll.get_an_integer() + assert len(w) == 1 + assert issubclass(w[0].category, RuntimeWarning) + assert "C function without declared return type called" in str(w[0].message) - @pytest.mark.xfail - def test_errcheck(self, dll): - import warnings - def errcheck(result, func, args): - assert result == -42 - assert type(result) is int - arg, = args - assert arg == -126 - assert type(arg) is int - return result - # - tf_b = dll.tf_b - tf_b.restype = c_byte - tf_b.argtypes = (c_byte,) - tf_b.errcheck = errcheck - assert tf_b(-126) == -42 - del tf_b.errcheck - with warnings.catch_warnings(record=True) as w: - dll.get_an_integer.argtypes = [] - dll.get_an_integer() - assert len(w) == 1 - assert issubclass(w[0].category, RuntimeWarning) - assert "C function without declared return type called" in str(w[0].message) + with warnings.catch_warnings(record=True) as w: + dll.get_an_integer.restype = None + dll.get_an_integer() + assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: - dll.get_an_integer.restype = None - dll.get_an_integer() - assert len(w) == 0 + warnings.resetwarnings() - warnings.resetwarnings() +def test_errno(dll): + test_errno = dll.test_errno + test_errno.restype = c_int + set_errno(42) + res = test_errno() + n = get_errno() + assert (res, n) == (42, 43) + set_errno(0) + assert get_errno() == 0 - def test_errno(self, dll): - test_errno = dll.test_errno - test_errno.restype = c_int - set_errno(42) - res = test_errno() - n = get_errno() - assert (res, n) == (42, 43) - set_errno(0) - assert get_errno() == 0 +def test_issue1655(dll): + def ret_list_p(icount): + def sz_array_p(obj, func, args): + assert ('.LP_c_int object' in repr(obj) or + '.LP_c_long object' in repr(obj)) + assert repr(args) in ("('testing!', c_int(4))", + "('testing!', c_long(4))") + assert args[icount].value == 4 + return [obj[i] for i in range(args[icount].value)] + return sz_array_p - def test_issue1655(self, dll): - def ret_list_p(icount): - def sz_array_p(obj, func, args): - assert ('.LP_c_int object' in repr(obj) or - '.LP_c_long object' in repr(obj)) - assert repr(args) in ("('testing!', c_int(4))", - "('testing!', c_long(4))") - assert args[icount].value == 4 - return [obj[i] for i in range(args[icount].value)] - return sz_array_p + get_data_prototype = CFUNCTYPE(POINTER(c_int), + c_char_p, POINTER(c_int)) + get_data_paramflag = ((1,), (2,)) + get_data_signature = ('test_issue1655', dll) - get_data_prototype = CFUNCTYPE(POINTER(c_int), - c_char_p, POINTER(c_int)) - get_data_paramflag = ((1,), (2,)) - get_data_signature = ('test_issue1655', dll) + get_data = get_data_prototype(get_data_signature, get_data_paramflag) + assert get_data('testing!') == 4 - get_data = get_data_prototype(get_data_signature, get_data_paramflag) - assert get_data('testing!') == 4 + get_data.errcheck = ret_list_p(1) + assert get_data('testing!') == [-1, -2, -3, -4] - get_data.errcheck = ret_list_p(1) - assert get_data('testing!') == [-1, -2, -3, -4] +def test_issue2533(tmpdir): + import cffi + ffi = cffi.FFI() + ffi.cdef("int **fetchme(void);") + ffi.set_source("_x_cffi", """ + int **fetchme(void) + { + static int a = 42; + static int *pa = &a; + return &pa; + } + """) + ffi.compile(verbose=True, tmpdir=str(tmpdir)) - def test_issue2533(self, tmpdir): - import cffi - ffi = cffi.FFI() - ffi.cdef("int **fetchme(void);") - ffi.set_source("_x_cffi", """ - int **fetchme(void) - { - static int a = 42; - static int *pa = &a; - return &pa; - } - """) - ffi.compile(verbose=True, tmpdir=str(tmpdir)) + import sys + sys.path.insert(0, str(tmpdir)) + try: + from _x_cffi import ffi, lib + finally: + sys.path.pop(0) + fetchme = ffi.addressof(lib, 'fetchme') + fetchme = int(ffi.cast("intptr_t", fetchme)) - import sys - sys.path.insert(0, str(tmpdir)) - try: - from _x_cffi import ffi, lib - finally: - sys.path.pop(0) - fetchme = ffi.addressof(lib, 'fetchme') - fetchme = int(ffi.cast("intptr_t", fetchme)) + FN = CFUNCTYPE(POINTER(POINTER(c_int))) + ff = cast(fetchme, FN) - FN = CFUNCTYPE(POINTER(POINTER(c_int))) - ff = cast(fetchme, FN) + g = ff() + assert g.contents.contents.value == 42 - g = ff() - assert g.contents.contents.value == 42 - - h = c_int(43) - g[0] = pointer(h) # used to crash here - assert g.contents.contents.value == 43 + h = c_int(43) + g[0] = pointer(h) # used to crash here + assert g.contents.contents.value == 43 From pypy.commits at gmail.com Wed Dec 12 05:52:52 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 12 Dec 2018 02:52:52 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: typos, fix whitespace and a test Message-ID: <5c10e884.1c69fb81.b5752.e32a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95453:0f144096d308 Date: 2018-12-11 12:13 +0100 http://bitbucket.org/pypy/pypy/changeset/0f144096d308/ Log: typos, fix whitespace and a test diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -442,6 +442,8 @@ return descr_binop, descr_rbinop descr_add, descr_radd = _make_generic_descr_binop('add') + + # XXX should support fast int version of rsub descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub') descr_mul, descr_rmul = _make_generic_descr_binop('mul') descr_and, descr_rand = _make_generic_descr_binop('and') @@ -481,10 +483,10 @@ raise oefmt(space.w_OverflowError, "shift count too large") return W_LongObject(self.num.lshift(shift)) - def _int_lshift(self, space, w_other): - if w_other < 0: + def _int_lshift(self, space, other): + if other < 0: raise oefmt(space.w_ValueError, "negative shift count") - return W_LongObject(self.num.lshift(w_other)) + return W_LongObject(self.num.lshift(other)) descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift) @@ -497,11 +499,11 @@ raise oefmt(space.w_OverflowError, "shift count too large") return newlong(space, self.num.rshift(shift)) - def _int_rshift(self, space, w_other): - if w_other < 0: + def _int_rshift(self, space, other): + if other < 0: raise oefmt(space.w_ValueError, "negative shift count") - return newlong(space, self.num.rshift(w_other)) + return newlong(space, self.num.rshift(other)) descr_rshift, descr_rrshift = _make_descr_binop(_rshift, _int_rshift) def _floordiv(self, space, w_other): @@ -512,9 +514,9 @@ "long division or modulo by zero") return newlong(space, z) - def _int_floordiv(self, space, w_other): + def _int_floordiv(self, space, other): try: - z = self.num.int_floordiv(w_other) + z = self.num.int_floordiv(other) except ZeroDivisionError: raise oefmt(space.w_ZeroDivisionError, "long division or modulo by zero") @@ -533,9 +535,9 @@ "long division or modulo by zero") return newlong(space, z) - def _int_mod(self, space, w_other): + def _int_mod(self, space, other): try: - z = self.num.int_mod(w_other) + z = self.num.int_mod(other) except ZeroDivisionError: raise oefmt(space.w_ZeroDivisionError, "long division or modulo by zero") @@ -549,15 +551,15 @@ raise oefmt(space.w_ZeroDivisionError, "long division or modulo by zero") return space.newtuple([newlong(space, div), newlong(space, mod)]) - - def _int_divmod(self, space, w_other): + + def _int_divmod(self, space, other): try: - div, mod = self.num.int_divmod(w_other) + div, mod = self.num.int_divmod(other) except ZeroDivisionError: raise oefmt(space.w_ZeroDivisionError, "long division or modulo by zero") return space.newtuple([newlong(space, div), newlong(space, mod)]) - + descr_divmod, descr_rdivmod = _make_descr_binop(_divmod, _int_divmod) diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -234,7 +234,7 @@ q, r = divmod(100L, 11) assert q == 9L assert r == 1L - + def test_format(self): assert repr(12345678901234567890) == '12345678901234567890L' assert str(12345678901234567890) == '12345678901234567890' diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -598,7 +598,7 @@ if not int_in_valid_range(other): # Fallback to Long. - return self.lt(rbigint.fromint(other)) + return self.le(rbigint.fromint(other)) return _x_int_lt(self, other, True) @@ -1199,7 +1199,7 @@ z._normalize() return z - lshift._always_inline_ = True # It's so fast that it's always benefitial. + lshift._always_inline_ = True # It's so fast that it's always beneficial. @jit.elidable def lqshift(self, int_other): @@ -1219,7 +1219,7 @@ z.setdigit(oldsize, accum) z._normalize() return z - lqshift._always_inline_ = True # It's so fast that it's always benefitial. + lqshift._always_inline_ = True # It's so fast that it's always beneficial. @jit.elidable def rshift(self, int_other, dont_invert=False): @@ -1262,7 +1262,7 @@ z.setdigit(0, z.digit(0)+1) z._normalize() return z - rshift._always_inline_ = 'try' # It's so fast that it's always benefitial. + rshift._always_inline_ = 'try' # It's so fast that it's always beneficial. @jit.elidable def rqshift(self, int_other): @@ -1287,7 +1287,7 @@ wordshift += 1 z._normalize() return z - rshift._always_inline_ = 'try' # It's so fast that it's always benefitial. + rshift._always_inline_ = 'try' # It's so fast that it's always beneficial. @jit.elidable def abs_rshift_and_mask(self, bigshiftcount, mask): diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -647,24 +647,22 @@ num = (x << y) + x f1 = rbigint.fromlong(num) nf1 = rbigint.fromlong(-num) - + for z in range(1, 31): - res1 = f1.lqshift(z).tolong() - res2 = f1.rqshift(z).tolong() - res3 = nf1.lqshift(z).tolong() - - + res1 = f1.lqshift(z).tolong() + res2 = f1.rqshift(z).tolong() + res3 = nf1.lqshift(z).tolong() + assert res1 == num << z assert res2 == num >> z assert res3 == -num << z - - + # Large digit for x in range((1 << SHIFT) - 10, (1 << SHIFT) + 10): f1 = rbigint.fromlong(x) - assert f1.rqshift(SHIFT).tolong() == x >> SHIFT + assert f1.rqshift(SHIFT).tolong() == x >> SHIFT assert f1.rqshift(SHIFT+1).tolong() == x >> (SHIFT+1) - + def test_from_list_n_bits(self): for x in ([3L ** 30L, 5L ** 20L, 7 ** 300] + [1L << i for i in range(130)] + diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -343,7 +343,7 @@ 'ulllong_lshift': LLOp(canfold=True), # args (r_ulonglonglong, int) 'ulllong_rshift': LLOp(canfold=True), # args (r_ulonglonglong, int) 'ulllong_xor': LLOp(canfold=True), - + 'cast_primitive': LLOp(canfold=True), 'cast_bool_to_int': LLOp(canfold=True), 'cast_bool_to_uint': LLOp(canfold=True), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -376,7 +376,7 @@ assert isinstance(x, r_ulonglonglong) assert is_valid_int(y) return r_ulonglonglong(x >> y) - + def op_same_as(x): return x diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py --- a/rpython/rtyper/rint.py +++ b/rpython/rtyper/rint.py @@ -482,7 +482,7 @@ if y == 0: raise ZeroDivisionError("unsigned longlonglong division") return ll_ulllong_py_div(x, y) - + # ---------- mod ---------- @jit.oopspec("int.py_mod(x, y)") @@ -562,7 +562,7 @@ if y == 0: raise ZeroDivisionError return ll_lllong_py_mod(x, y) - + @jit.dont_look_inside def ll_ulllong_py_mod(x, y): return llop.ulllong_mod(UnsignedLongLongLong, x, y) From pypy.commits at gmail.com Wed Dec 12 05:52:54 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 12 Dec 2018 02:52:54 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: slightly more careful testing of three-operand pow Message-ID: <5c10e886.1c69fb81.7d2b4.b791@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95454:0e5c477cc330 Date: 2018-12-12 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/0e5c477cc330/ Log: slightly more careful testing of three-operand pow diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -189,36 +189,43 @@ def test_pow(self): for op1 in gen_signs(long_vals_not_too_big): + rl_op1 = rbigint.fromlong(op1) for op2 in [0, 1, 2, 8, 9, 10, 11]: - rl_op1 = rbigint.fromlong(op1) rl_op2 = rbigint.fromint(op2) r1 = rl_op1.pow(rl_op2) r2 = op1 ** op2 assert r1.tolong() == r2 - r3 = rl_op1.int_pow(op2, rbigint.fromint(1000)) - r4 = pow(op1, op2, 1000) - print op1, op2 - assert r3.tolong() == r4 + for op3 in gen_signs([1, 2, 5, 1000, 12312312312312235659969696l]): + if not op3: + continue + print op1, op2, op3 + r3 = rl_op1.pow(rl_op2, rbigint.fromlong(op3)) + r4 = pow(op1, op2, op3) + assert r3.tolong() == r4 def test_int_pow(self): - for op1 in gen_signs(long_vals_not_too_big[:-2]): - for op2 in [0, 1, 2, 8, 9, 10, 11]: - rl_op1 = rbigint.fromlong(op1) + for op1 in gen_signs(long_vals_not_too_big): + rl_op1 = rbigint.fromlong(op1) + for op2 in [0, 1, 2, 8, 9, 10, 11, 127, 128, 129]: r1 = rl_op1.int_pow(op2) r2 = op1 ** op2 assert r1.tolong() == r2 - r3 = rl_op1.int_pow(op2, rbigint.fromint(1000)) - r4 = pow(op1, op2, 1000) - print op1, op2 - assert r3.tolong() == r4 + for op3 in gen_signs(long_vals_not_too_big): + if not op3: + continue + r3 = rl_op1.int_pow(op2, rbigint.fromlong(op3)) + r4 = pow(op1, op2, op3) + print op1, op2, op3 + assert r3.tolong() == r4 def test_pow_raises(self): r1 = rbigint.fromint(2) r0 = rbigint.fromint(0) py.test.raises(ValueError, r1.int_pow, 2, r0) py.test.raises(ValueError, r1.pow, r1, r0) + def test_touint(self): result = r_uint(sys.maxint + 42) rl = rbigint.fromint(sys.maxint).add(rbigint.fromint(42)) @@ -865,7 +872,7 @@ y += randint(1, 1 << 60) if y > x: x <<= 100 - + f1 = rbigint.fromlong(x) f2 = rbigint.fromlong(y) div, rem = lobj._x_divrem(f1, f2) From pypy.commits at gmail.com Wed Dec 12 15:57:52 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Dec 2018 12:57:52 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Modernise test_callbacks Message-ID: <5c117650.1c69fb81.ecd85.197e@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95456:c8caf9dcef18 Date: 2018-12-12 20:38 +0000 http://bitbucket.org/pypy/pypy/changeset/c8caf9dcef18/ Log: Modernise test_callbacks diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py @@ -1,124 +1,84 @@ +import pytest + +import math from ctypes import * -import pytest from .support import BaseCTypesTestChecker -class TestCallbacks(BaseCTypesTestChecker): - functype = CFUNCTYPE - - def callback(self, *args): - self.got_args = args - return args[-1] - - def check_type(self, typ, arg): - unwrapped_types = { - c_float: (float,), - c_double: (float,), - c_char: (str,), - c_char_p: (str,), - c_uint: (int, long), - c_ulong: (int, long), - } - - PROTO = self.functype.im_func(typ, typ) - cfunc = PROTO(self.callback) - result = cfunc(arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (arg,) - assert result == arg - - result2 = cfunc(typ(arg)) - assert type(result2) in unwrapped_types.get(typ, (int, long)) - - PROTO = self.functype.im_func(typ, c_byte, typ) - result = PROTO(self.callback)(-3, arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (-3, arg) - assert result == arg - - ################ - - def test_byte(self): - self.check_type(c_byte, 42) - self.check_type(c_byte, -42) - - def test_ubyte(self): - self.check_type(c_ubyte, 42) - - def test_short(self): - self.check_type(c_short, 42) - self.check_type(c_short, -42) - - def test_ushort(self): - self.check_type(c_ushort, 42) - - def test_int(self): - self.check_type(c_int, 42) - self.check_type(c_int, -42) - - def test_uint(self): - self.check_type(c_uint, 42) - - def test_long(self): - self.check_type(c_long, 42) - self.check_type(c_long, -42) - - def test_ulong(self): - self.check_type(c_ulong, 42) - - def test_longlong(self): - self.check_type(c_longlong, 42) - self.check_type(c_longlong, -42) - - def test_ulonglong(self): - self.check_type(c_ulonglong, 42) - - def test_float(self): - # only almost equal: double -> float -> double - import math - self.check_type(c_float, math.e) - self.check_type(c_float, -math.e) - - def test_double(self): - self.check_type(c_double, 3.14) - self.check_type(c_double, -3.14) - - def test_char(self): - self.check_type(c_char, "x") - self.check_type(c_char, "a") - - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. -## def test_char_p(self): -## self.check_type(c_char_p, "abc") -## self.check_type(c_char_p, "def") - - - @pytest.mark.xfail( - reason="we are less strict about callback return type sanity") - def test_unsupported_restype_1(self): - # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. - # POINTER(c_double), for example, is not supported. - - prototype = self.functype.im_func(POINTER(c_double)) - # The type is checked when the prototype is called - with pytest.raises(TypeError): - prototype(lambda: None) - +functypes = [CFUNCTYPE] try: - WINFUNCTYPE + functypes.append(WINFUNCTYPE) except NameError: pass -else: - class TestStdcallCallbacks(TestCallbacks): - functype = WINFUNCTYPE -################################################################ + +def callback(*args): + callback.got_args = args + return args[-1] + +unwrapped_types = { + c_float: (float,), + c_double: (float,), + c_char: (str,), + c_char_p: (str,), + c_uint: (int, long), + c_ulong: (int, long), + } + + at pytest.mark.parametrize("typ, arg", [ + (c_byte, 42), + (c_byte, -42), + (c_ubyte, 42), + (c_short, 42), + (c_short, -42), + (c_ushort, 42), + (c_int, 42), + (c_int, -42), + (c_uint, 42), + (c_long, 42), + (c_long, -42), + (c_ulong, 42), + (c_longlong, 42), + (c_longlong, -42), + (c_ulonglong, 42), + (c_float, math.e), # only almost equal: double -> float -> double + (c_float, -math.e), + (c_double, 3.14), + (c_double, -3.14), + (c_char, "x"), + (c_char, "a"), +]) + at pytest.mark.parametrize('functype', functypes) +def test_types(typ, arg, functype): + PROTO = functype(typ, typ) + cfunc = PROTO(callback) + result = cfunc(arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (arg,) + assert result == arg + + result2 = cfunc(typ(arg)) + assert type(result2) in unwrapped_types.get(typ, (int, long)) + + PROTO = functype(typ, c_byte, typ) + result = PROTO(callback)(-3, arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (-3, arg) + assert result == arg + + at pytest.mark.parametrize('functype', functypes) +def test_unsupported_restype_1(functype): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. + + prototype = functype(POINTER(c_double)) + # The type is checked when the prototype is called + with pytest.raises(TypeError): + prototype(lambda: None) def test_callback_with_struct_argument(): @@ -127,16 +87,14 @@ ("right", c_int), ("bottom", c_int)] proto = CFUNCTYPE(c_int, RECT) + def callback(point): point.left *= -1 - return point.left+point.top+point.right+point.bottom + return point.left + point.top + point.right + point.bottom cbp = proto(callback) - - rect = RECT(-1000,100,10,1) - + rect = RECT(-1000, 100, 10, 1) res = cbp(rect) - assert res == 1111 assert rect.left == -1000 # must not have been changed! @@ -146,11 +104,12 @@ ("right", c_long), ("bottom", c_long)] proto = CFUNCTYPE(c_int, RECT) + def callback(point): - return point.left+point.top+point.right+point.bottom + return point.left + point.top + point.right + point.bottom cbp = proto(callback) - rect = RECT(1000,100,10,1) + rect = RECT(1000, 100, 10, 1) call_callback_with_rect = dll.call_callback_with_rect call_callback_with_rect.restype = c_int From pypy.commits at gmail.com Wed Dec 12 17:14:34 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Dec 2018 14:14:34 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move ctypes_tests to extra_tests/ Message-ID: <5c11884a.1c69fb81.65ce6.17f3@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95464:d752b7ef5622 Date: 2018-12-12 21:10 +0000 http://bitbucket.org/pypy/pypy/changeset/d752b7ef5622/ Log: Move ctypes_tests to extra_tests/ diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/extra_tests/ctypes_tests/README rename from pypy/module/test_lib_pypy/ctypes_tests/README rename to extra_tests/ctypes_tests/README diff --git a/pypy/module/test_lib_pypy/ctypes_tests/__init__.py b/extra_tests/ctypes_tests/__init__.py rename from pypy/module/test_lib_pypy/ctypes_tests/__init__.py rename to extra_tests/ctypes_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/extra_tests/ctypes_tests/_ctypes_test.c rename from pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c rename to extra_tests/ctypes_tests/_ctypes_test.c diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py rename from pypy/module/test_lib_pypy/ctypes_tests/conftest.py rename to extra_tests/ctypes_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/description.txt b/extra_tests/ctypes_tests/description.txt rename from pypy/module/test_lib_pypy/ctypes_tests/description.txt rename to extra_tests/ctypes_tests/description.txt diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/extra_tests/ctypes_tests/support.py rename from pypy/module/test_lib_pypy/ctypes_tests/support.py rename to extra_tests/ctypes_tests/support.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_anon.py rename to extra_tests/ctypes_tests/test_anon.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_array.py rename to extra_tests/ctypes_tests/test_array.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_base.py rename to extra_tests/ctypes_tests/test_base.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/extra_tests/ctypes_tests/test_bitfields.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py rename to extra_tests/ctypes_tests/test_bitfields.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py rename to extra_tests/ctypes_tests/test_buffers.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/extra_tests/ctypes_tests/test_callback_traceback.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py rename to extra_tests/ctypes_tests/test_callback_traceback.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py rename to extra_tests/ctypes_tests/test_callbacks.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/extra_tests/ctypes_tests/test_cast.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_cast.py rename to extra_tests/ctypes_tests/test_cast.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py b/extra_tests/ctypes_tests/test_commethods.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py rename to extra_tests/ctypes_tests/test_commethods.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/extra_tests/ctypes_tests/test_errno.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_errno.py rename to extra_tests/ctypes_tests/test_errno.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_extra.py rename to extra_tests/ctypes_tests/test_extra.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py b/extra_tests/ctypes_tests/test_funcptr.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_funcptr.py rename to extra_tests/ctypes_tests/test_funcptr.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/extra_tests/ctypes_tests/test_functions.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_functions.py rename to extra_tests/ctypes_tests/test_functions.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py b/extra_tests/ctypes_tests/test_guess_argtypes.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py rename to extra_tests/ctypes_tests/test_guess_argtypes.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py b/extra_tests/ctypes_tests/test_keepalive.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py rename to extra_tests/ctypes_tests/test_keepalive.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_loading.py b/extra_tests/ctypes_tests/test_loading.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_loading.py rename to extra_tests/ctypes_tests/test_loading.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/extra_tests/ctypes_tests/test_numbers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py rename to extra_tests/ctypes_tests/test_numbers.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py b/extra_tests/ctypes_tests/test_parameters.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_parameters.py rename to extra_tests/ctypes_tests/test_parameters.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/extra_tests/ctypes_tests/test_pointers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py rename to extra_tests/ctypes_tests/test_pointers.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py b/extra_tests/ctypes_tests/test_prototypes.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_prototypes.py rename to extra_tests/ctypes_tests/test_prototypes.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_structures.py rename to extra_tests/ctypes_tests/test_structures.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_unions.py b/extra_tests/ctypes_tests/test_unions.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_unions.py rename to extra_tests/ctypes_tests/test_unions.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_values.py b/extra_tests/ctypes_tests/test_values.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_values.py rename to extra_tests/ctypes_tests/test_values.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_win32.py b/extra_tests/ctypes_tests/test_win32.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_win32.py rename to extra_tests/ctypes_tests/test_win32.py From pypy.commits at gmail.com Wed Dec 12 17:14:36 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Dec 2018 14:14:36 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Compatibility with pytest 4.* Message-ID: <5c11884c.1c69fb81.ca700.2343@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95465:261405fad710 Date: 2018-12-12 21:20 +0000 http://bitbucket.org/pypy/pypy/changeset/261405fad710/ Log: Compatibility with pytest 4.* diff --git a/extra_tests/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py --- a/extra_tests/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -85,8 +85,7 @@ return outputfilename # end copy -def compile_so_file(): - udir = pytest.ensuretemp('_ctypes_test') +def compile_so_file(udir): cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': @@ -97,8 +96,9 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) @pytest.fixture(scope='session') -def sofile(): - return str(compile_so_file()) +def sofile(tmpdir_factory): + udir = tmpdir_factory.mktemp('_ctypes_test') + return str(compile_so_file(udir)) @pytest.fixture def dll(sofile): From pypy.commits at gmail.com Wed Dec 12 17:14:38 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Dec 2018 14:14:38 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Allow (most) ctypes tests to run on CPython and add @pytest.mark.pypy_only Message-ID: <5c11884e.1c69fb81.b0079.19e3@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95466:74de35e03dac Date: 2018-12-12 22:13 +0000 http://bitbucket.org/pypy/pypy/changeset/74de35e03dac/ Log: Allow (most) ctypes tests to run on CPython and add @pytest.mark.pypy_only diff --git a/extra_tests/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py --- a/extra_tests/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -3,10 +3,6 @@ import sys import os -def pytest_ignore_collect(path): - if '__pypy__' not in sys.builtin_module_names: - return True - # XXX: copied from pypy/tool/cpyext/extbuild.py if os.name != 'nt': so_ext = 'so' diff --git a/extra_tests/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py --- a/extra_tests/ctypes_tests/test_anon.py +++ b/extra_tests/ctypes_tests/test_anon.py @@ -1,6 +1,7 @@ import pytest from ctypes import * + at pytest.mark.pypy_only def test_nested(): class ANON_S(Structure): _fields_ = [("a", c_int)] diff --git a/extra_tests/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py --- a/extra_tests/ctypes_tests/test_base.py +++ b/extra_tests/ctypes_tests/test_base.py @@ -1,6 +1,7 @@ +import pytest from ctypes import * -# WhiteBoxTests +pytestmark = pytest.mark.pypy_only def test_pointer(): p = pointer(pointer(c_int(2))) diff --git a/extra_tests/ctypes_tests/test_functions.py b/extra_tests/ctypes_tests/test_functions.py --- a/extra_tests/ctypes_tests/test_functions.py +++ b/extra_tests/ctypes_tests/test_functions.py @@ -57,6 +57,7 @@ result = f("abcd", ord("b")) assert result == "bcd" + at pytest.mark.pypy_only def test_keepalive_buffers(monkeypatch, dll): import gc f = dll.my_strchr @@ -111,6 +112,7 @@ u = dll.ret_un_func(a[1]) assert u.y == 33 * 10000 + at pytest.mark.pypy_only def test_cache_funcptr(dll): tf_b = dll.tf_b tf_b.restype = c_byte diff --git a/extra_tests/ctypes_tests/test_guess_argtypes.py b/extra_tests/ctypes_tests/test_guess_argtypes.py --- a/extra_tests/ctypes_tests/test_guess_argtypes.py +++ b/extra_tests/ctypes_tests/test_guess_argtypes.py @@ -1,15 +1,14 @@ """ This test checks whether args wrapping behavior is correct """ -import py +import pytest import sys from ctypes import * + at pytest.mark.pypy_only def test_wrap_args(): - if not hasattr(sys, 'pypy_translation_info'): - py.test.skip("pypy white-box test") - from _ctypes.function import CFuncPtr + from _ctypes import CFuncPtr def guess(value): _, cobj, ctype = CFuncPtr._conv_param(None, value) @@ -33,7 +32,7 @@ def test_guess_unicode(dll): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': - py.test.skip("CPython segfaults: see http://bugs.python.org/issue5203") + pytest.skip("CPython segfaults: see http://bugs.python.org/issue5203") wcslen = dll.my_wcslen text = u"Some long unicode string" assert wcslen(text) == len(text) diff --git a/extra_tests/ctypes_tests/test_keepalive.py b/extra_tests/ctypes_tests/test_keepalive.py --- a/extra_tests/ctypes_tests/test_keepalive.py +++ b/extra_tests/ctypes_tests/test_keepalive.py @@ -104,7 +104,7 @@ assert p._objects.keys() == ['1'] assert p._objects['1'].value == 3 - at pytest.mark.skipif("'__pypy__' not in sys.builtin_module_names") + at pytest.mark.pypy_only def test_primitive(): assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None diff --git a/extra_tests/ctypes_tests/test_pointers.py b/extra_tests/ctypes_tests/test_pointers.py --- a/extra_tests/ctypes_tests/test_pointers.py +++ b/extra_tests/ctypes_tests/test_pointers.py @@ -1,6 +1,7 @@ import pytest from ctypes import * + at pytest.mark.pypy_only def test_get_ffi_argtype(): P = POINTER(c_int) ffitype = P.get_ffi_argtype() @@ -61,6 +62,7 @@ for i in [0, 1, 4, 1444, -10293]: assert cast(byref(c, i), c_void_p).value == base + i + at pytest.mark.pypy_only def test_issue2813_fix(): class C(Structure): pass @@ -70,6 +72,7 @@ assert C.get_ffi_argtype() is ffitype assert ffitype.sizeof() == sizeof(c_int) + at pytest.mark.pypy_only def test_issue2813_cant_change_fields_after_get_ffi_argtype(): class C(Structure): pass From pypy.commits at gmail.com Thu Dec 13 02:52:55 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 12 Dec 2018 23:52:55 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 into branch Message-ID: <5c120fd7.1c69fb81.e292a.4ff4@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95467:4626a6708b2b Date: 2018-12-09 12:22 +0200 http://bitbucket.org/pypy/pypy/changeset/4626a6708b2b/ Log: merge py3.5 into branch diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py --- a/extra_tests/test_decimal.py +++ b/extra_tests/test_decimal.py @@ -4,7 +4,7 @@ import pickle import sys -from support import import_fresh_module +from .support import import_fresh_module C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1542,7 +1542,7 @@ def _verify(ffi, module_name, preamble, *args, **kwds): # FOR TESTS ONLY - from testing.udir import udir + from .testing.udir import udir import imp assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -55,3 +55,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -386,7 +386,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -604,8 +604,14 @@ warnoptions = pythonwarnings.split(',') + warnoptions if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in str(e.value) + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in str(excinfo.value) - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in str(e.value) +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in str(excinfo.value) - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -97,8 +97,8 @@ w_obj = space.call_function(w_inttype, w_obj) return w_obj - @unwrap_spec(nbytes=int, byteorder='text', signed=bool) - def descr_to_bytes(self, space, nbytes, byteorder, signed=False): + @unwrap_spec(length=int, byteorder='text', signed=bool) + def descr_to_bytes(self, space, length, byteorder, signed=False): """to_bytes(...) int.to_bytes(length, byteorder, *, signed=False) -> bytes @@ -121,7 +121,7 @@ """ bigint = space.bigint_w(self) try: - byte_string = bigint.tobytes(nbytes, byteorder=byteorder, + byte_string = bigint.tobytes(length, byteorder=byteorder, signed=signed) except InvalidEndiannessError: raise oefmt(space.w_ValueError, diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -379,6 +379,7 @@ assert (-8388608).to_bytes(3, 'little', signed=True) == b'\x00\x00\x80' raises(OverflowError, (-5).to_bytes, 1, 'big') raises(ValueError, (-5).to_bytes, 1, 'foo') + assert 65535 .to_bytes(length=2, byteorder='big') == b'\xff\xff' def test_negative_zero(self): x = eval("-self._long(0)") diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -44,7 +44,7 @@ if config.translation.jit_profiler == "oprofile": from rpython.jit.backend.x86 import oprofile if not oprofile.OPROFILE_AVAILABLE: - log.WARNING('oprofile support was explicitly enabled, but oprofile headers seem not to be available') + raise Exception('oprofile support was explicitly enabled, but oprofile headers seem not to be available') profile_agent = oprofile.OProfileAgent() self.with_threads = config.translation.thread diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -11,7 +11,7 @@ from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - AbstractResOp, GuardResOp + GuardResOp from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -6,7 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rlib.rarithmetic import intmask @@ -44,8 +44,9 @@ if b.has_lower and b.has_upper and b.lower == b.upper: self.make_constant_int(box, b.lower) - if isinstance(box, AbstractResOp): - dispatch_bounds_ops(self, box) + box1 = self.optimizer.as_operation(box) + if box1 is not None: + dispatch_bounds_ops(self, box1) def _optimize_guard_true_false_value(self, op): return self.emit(op) @@ -126,10 +127,11 @@ v1, v2 = v2, v1 # if both are constant, the pure optimization will deal with it if v2.is_constant() and not v1.is_constant(): - if not self.optimizer.is_inputarg(arg1): + arg1 = self.optimizer.as_operation(arg1) + if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = arg1.getarg(0) - prod_arg2 = arg1.getarg(1) + prod_arg1 = self.get_box_replacement(arg1.getarg(0)) + prod_arg2 = self.get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -279,6 +279,7 @@ self.quasi_immutable_deps = None self.replaces_guard = {} self._newoperations = [] + self._emittedoperations = {} self.optimizer = self self.optpure = None self.optheap = None @@ -292,11 +293,6 @@ self.set_optimizations(optimizations) self.setup() - def init_inparg_dict_from(self, lst): - self.inparg_dict = {} - for box in lst: - self.inparg_dict[box] = None - def set_optimizations(self, optimizations): if optimizations: self.first_optimization = optimizations[0] @@ -384,9 +380,12 @@ return info.force_box(op, optforce) return op - def is_inputarg(self, op): - return True - return op in self.inparg_dict + def as_operation(self, op): + # You should never check "isinstance(op, AbstractResOp" directly. + # Instead, use this helper. + if isinstance(op, AbstractResOp) and op in self._emittedoperations: + return op + return None def get_constant_box(self, box): box = self.get_box_replacement(box) @@ -404,6 +403,7 @@ def clear_newoperations(self): self._newoperations = [] + self._emittedoperations = {} def make_equal_to(self, op, newop): op = self.get_box_replacement(op) @@ -631,6 +631,7 @@ self._last_guard_op = None self._really_emitted_operation = op self._newoperations.append(op) + self._emittedoperations[op] = None def emit_guard_operation(self, op, pendingfields): guard_op = op # self.replace_op_with(op, op.getopnum()) @@ -675,6 +676,7 @@ return newop = self.replace_op_with_no_ovf(op) self._newoperations[-1] = newop + self._emittedoperations[newop] = None def replace_op_with_no_ovf(self, op): if op.getopnum() == rop.INT_MUL_OVF: @@ -719,6 +721,7 @@ new_descr = new_op.getdescr() new_descr.copy_all_attributes_from(old_descr) self._newoperations[old_op_pos] = new_op + self._emittedoperations[new_op] = None def store_final_boxes_in_guard(self, op, pendingfields): assert pendingfields is not None diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers, AbstractResOp + OpHelpers from rpython.rlib.rarithmetic import highest_bit from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper import rclass @@ -389,6 +389,8 @@ def optimize_GUARD_SUBCLASS(self, op): info = self.getptrinfo(op.getarg(0)) optimizer = self.optimizer + # must raise 'InvalidLoop' in all cases where 'info' shows the + # class cannot possibly match (see test_issue2926) if info and info.is_constant(): c = self.get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.ts.cls_of_box(c).getint() @@ -398,13 +400,29 @@ if info is not None and info.is_about_object(): known_class = info.get_known_class(optimizer.cpu) if known_class: + # Class of 'info' is exactly 'known_class'. + # We know statically if the 'guard_subclass' will pass or fail. if optimizer._check_subclass(known_class.getint(), op.getarg(1).getint()): return + else: + raise InvalidLoop( + "GUARD_SUBCLASS(known_class) proven to always fail") elif info.get_descr() is not None: - if optimizer._check_subclass(info.get_descr().get_vtable(), + # Class of 'info' is either get_descr() or a subclass of it. + # We're keeping the 'guard_subclass' at runtime only in the + # case where get_descr() is some strict parent class of + # the argument to 'guard_subclass'. + info_base_descr = info.get_descr().get_vtable() + if optimizer._check_subclass(info_base_descr, op.getarg(1).getint()): - return + return # guard_subclass always passing + elif optimizer._check_subclass(op.getarg(1).getint(), + info_base_descr): + pass # don't know, must keep the 'guard_subclass' + else: + raise InvalidLoop( + "GUARD_SUBCLASS(base_class) proven to always fail") return self.emit(op) def optimize_GUARD_NONNULL(self, op): @@ -490,11 +508,11 @@ def postprocess_GUARD_TRUE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_TRUE): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_TRUE: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0) + self.pure_from_args(rop.INT_IS_ZERO, [box1.getarg(0)], CONST_0) self.make_constant(box, CONST_1) def optimize_GUARD_FALSE(self, op): @@ -502,11 +520,11 @@ def postprocess_GUARD_FALSE(self, op): box = self.get_box_replacement(op.getarg(0)) - if (isinstance(box, AbstractResOp) and - box.getopnum() == rop.INT_IS_ZERO): + box1 = self.optimizer.as_operation(box) + if box1 is not None and box1.getopnum() == rop.INT_IS_ZERO: # we can't use the (current) range analysis for this because # "anything but 0" is not a valid range - self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1) + self.pure_from_args(rop.INT_IS_TRUE, [box1.getarg(0)], CONST_1) self.make_constant(box, CONST_0) def optimize_ASSERT_NOT_NONE(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -4017,7 +4017,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - ifoo = int_add(i5, 1) jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -74,7 +74,7 @@ print "Loop:" print '\n'.join([str(o) for o in loop.operations]) print - if expected_short: + if expected_short or getattr(info, 'short_preamble', None): print "Short Preamble:" short = info.short_preamble print '\n'.join([str(o) for o in short]) @@ -1300,7 +1300,7 @@ preamble = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -1310,7 +1310,7 @@ expected = """ [i0, p1, p3] i28 = int_add(i0, 1) - i29 = int_add(i28, 1) + i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) setfield_gc(p30, i28, descr=valuedescr) setfield_gc(p3, p30, descr=nextdescr) @@ -6392,7 +6392,6 @@ strsetitem(p3, i2, i0) i5 = int_add(i2, 1) strsetitem(p3, i5, i1) - i6 = int_add(i5, 1) # will be killed by the backend jump(i1, i0, p3) """ self.optimize_strunicode_loop(ops, expected, expected) @@ -9063,6 +9062,7 @@ self.optimize_loop(ops, expected) def test_same_as_preserves_info_in_the_preamble_2(self): + py.test.xfail("less efficient loop, investigate") ops = """ [i0, p0] ifoo = getfield_gc_i(p0, descr=valuedescr) @@ -9499,5 +9499,25 @@ """ self.optimize_loop(ops, expected) + def test_issue2904(self): + # we don't store advanced virtualstate information like "i1 = i2 + 1", + # which means that the following loop, when unrolled, cannot be + # optimized based on the knowledge that "i1 = i2 + 1" from the + # preamble---we can't use that knowledge. After the fix, we get + # the value "i2 + 1" passed as a third argument, possibly different + # from "i1". + ops = """ + [i1, i2] + guard_value(i1, 10) [] + i3 = int_add(i2, 1) + jump(i3, i2) + """ + expected = """ + [i1, i2, i3] + guard_value(i1, 10) [] + jump(i3, i2, i3) + """ + self.optimize_loop(ops, expected) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -10,8 +10,7 @@ from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo from rpython.jit.metainterp.optimizeopt.virtualstate import ( VirtualStateConstructor, VirtualStatesCantMatch) -from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp from rpython.jit.metainterp import compile from rpython.rlib.debug import debug_print, debug_start, debug_stop,\ have_debug_prints @@ -22,7 +21,6 @@ if self.optunroll.short_preamble_producer is None: assert False # unreachable code op = preamble_op.op - self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, preamble_op.preamble_op, self) @@ -144,7 +142,6 @@ except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} - self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) @@ -431,8 +428,9 @@ for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): - if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): - info = self.optimizer.getinfo(arg.getarg(0)) + arg1 = self.optimizer.as_operation(arg) + if arg1 is not None and rop.is_same_as(arg1.opnum): + info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py --- a/rpython/jit/metainterp/optimizeopt/vstring.py +++ b/rpython/jit/metainterp/optimizeopt/vstring.py @@ -6,8 +6,7 @@ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation,\ - AbstractResOp +from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.optimizeopt import info from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.unroll import unrolling_iterable diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,3 +4702,112 @@ res = self.meta_interp(f, [10]) assert res == f(10) + def test_cached_info_missing(self): + py.test.skip("XXX hitting a non-translated assert in optimizeopt/heap.py, but seems not to hurt the rest") + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + height = IntVal(height) + c = IntVal(0) + h = height + total = IntVal(0) + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) + if c.intval >= iterations: + return total.intval + h = height + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2904(self): + driver = JitDriver(greens = [], + reds=['iterations', 'total', 'c', 'height', 'h']) + + def f(height, iterations): + set_param(driver, 'threshold', 4) + set_param(driver, 'trace_eagerness', 1) + c = 0 + h = height + total = 0 + + while True: + driver.jit_merge_point(iterations=iterations, + total=total, c=c, height=height, h=h) + if h != 0: + h = h - 1 + total = total + 1 + else: + c = c + 1 + if c >= iterations: + return total + h = height - 1 + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) + + def test_issue2926(self): + driver = JitDriver(greens = [], reds=['i', 'total', 'p']) + + class Base(object): + def do_stuff(self): + return 1000 + class Int(Base): + def __init__(self, intval): + self.intval = intval + def do_stuff(self): + return self.intval + class SubInt(Int): + pass + class Float(Base): + def __init__(self, floatval): + self.floatval = floatval + def do_stuff(self): + return int(self.floatval) + + prebuilt = [Int(i) for i in range(10)] + + @dont_look_inside + def forget_intbounds(i): + return i + + @dont_look_inside + def escape(p): + pass + + def f(i): + total = 0 + p = Base() + while True: + driver.jit_merge_point(i=i, total=total, p=p) + #print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', i + if i == 13: + break + total += p.do_stuff() + j = forget_intbounds(i) + if j < 10: # initial loop + p = prebuilt[i] + p.intval = j + elif j < 12: + p = Int(i) + else: + p = Float(3.14) + escape(p) + i += 1 + return total + + res = self.meta_interp(f, [0]) + assert res == f(0) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -73,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -192,6 +192,8 @@ # ____________________________________________________________ + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +376,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1644,7 +1647,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1843,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2249,7 +2253,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2493,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3000,6 +3005,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,6 +1,7 @@ import sys import time +from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rarithmetic import is_valid_int, r_longlong @@ -75,6 +76,7 @@ _stop_colors = "" @always_inline + at enforceargs(str, bool) def debug_start(category, timestamp=False): """ Start a PYPYLOG section. @@ -85,6 +87,7 @@ return _debug_start(category, timestamp) @always_inline + at enforceargs(str, bool) def debug_stop(category, timestamp=False): """ Stop a PYPYLOG section. See debug_start for docs about timestamp diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -707,8 +707,8 @@ address.unlock() errno = _c.geterrno() timeout = self.timeout - if (timeout > 0.0 and res < 0 and - errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): + if (timeout > 0.0 and res < 0 and + errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): tv = rffi.make(_c.timeval) rffi.setintfield(tv, 'c_tv_sec', int(timeout)) rffi.setintfield(tv, 'c_tv_usec', diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -136,7 +136,10 @@ void = lltype.nullptr(rffi.VOIDP.TO) result = -1.0 if HAVE_GETTIMEOFDAY: - with lltype.scoped_alloc(TIMEVAL) as t: + # NB: can't use lltype.scoped_malloc, because that will allocate the + # with handler in the GC, but we want to use time.time from gc.collect! + t = lltype.malloc(TIMEVAL, flavor='raw') + try: errcode = -1 if GETTIMEOFDAY_NO_TZ: errcode = c_gettimeofday(t) @@ -145,13 +148,18 @@ if rffi.cast(rffi.LONG, errcode) == 0: result = decode_timeval(t) + finally: + lltype.free(t, flavor='raw') if result != -1: return result else: # assume using ftime(3) - with lltype.scoped_alloc(TIMEB) as t: + t = lltype.malloc(TIMEB, flavor='raw') + try: c_ftime(t) result = (float(intmask(t.c_time)) + float(intmask(t.c_millitm)) * 0.001) + finally: + lltype.free(t, flavor='raw') return result return float(c_time(void)) diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -179,7 +179,7 @@ t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) - assert total0 + total == 10 + assert total0 + total == 9 def test_loop(): l = [10, 12, 15, 1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -521,11 +521,9 @@ assert path.check(file=0) def test_debug_start_stop_timestamp(self): - import sys - import time from rpython.rlib.rtimer import read_timestamp def entry_point(argv): - timestamp = int(argv[1]) + timestamp = bool(int(argv[1])) ts1 = debug_start("foo", timestamp=timestamp) ts2 = read_timestamp() ts3 = debug_stop("foo", timestamp=timestamp) From pypy.commits at gmail.com Thu Dec 13 02:52:58 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 12 Dec 2018 23:52:58 -0800 (PST) Subject: [pypy-commit] pypy default: disable latex document generation Message-ID: <5c120fda.1c69fb81.10c6f.4806@mx.google.com> Author: Matti Picus Branch: Changeset: r95468:40929a5559b0 Date: 2018-12-13 09:51 +0200 http://bitbucket.org/pypy/pypy/changeset/40929a5559b0/ Log: disable latex document generation diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -192,10 +192,10 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('temp_index', 'PyPy.tex', u'PyPy Documentation', - u'The PyPy Project', 'manual'), -] +#latex_documents = [ +# ('temp_index', 'PyPy.tex', u'PyPy Documentation', +# u'The PyPy Project', 'manual'), +#] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -212,7 +212,7 @@ #latex_appendices = [] # If false, no module index is generated. -latex_use_modindex = False +#latex_use_modindex = False # Example configuration for intersphinx: refer to the Python standard library. From pypy.commits at gmail.com Thu Dec 13 03:57:40 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 00:57:40 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: consistent naming of method arguments Message-ID: <5c121f04.1c69fb81.cb795.b9fe@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95469:1a203e36cef8 Date: 2018-12-12 23:07 +0100 http://bitbucket.org/pypy/pypy/changeset/1a203e36cef8/ Log: consistent naming of method arguments diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -530,22 +530,22 @@ return True @jit.elidable - def int_eq(self, other): + def int_eq(self, iother): """ eq with int """ - if not int_in_valid_range(other): + if not int_in_valid_range(iother): # Fallback to Long. - return self.eq(rbigint.fromint(other)) + return self.eq(rbigint.fromint(iother)) if self.numdigits() > 1: return False - return (self.sign * self.digit(0)) == other + return (self.sign * self.digit(0)) == iother def ne(self, other): return not self.eq(other) - def int_ne(self, other): - return not self.int_eq(other) + def int_ne(self, iother): + return not self.int_eq(iother) @jit.elidable def lt(self, other): @@ -583,38 +583,38 @@ return False @jit.elidable - def int_lt(self, other): + def int_lt(self, iother): """ lt where other is an int """ - if not int_in_valid_range(other): + if not int_in_valid_range(iother): # Fallback to Long. - return self.lt(rbigint.fromint(other)) - - return _x_int_lt(self, other, False) + return self.lt(rbigint.fromint(iother)) + + return _x_int_lt(self, iother, False) def le(self, other): return not other.lt(self) - def int_le(self, other): - """ le where other is an int """ - - if not int_in_valid_range(other): + def int_le(self, iother): + """ le where iother is an int """ + + if not int_in_valid_range(iother): # Fallback to Long. - return self.le(rbigint.fromint(other)) - - return _x_int_lt(self, other, True) + return self.le(rbigint.fromint(iother)) + + return _x_int_lt(self, iother, True) def gt(self, other): return other.lt(self) - def int_gt(self, other): - return not self.int_le(other) + def int_gt(self, iother): + return not self.int_le(iother) def ge(self, other): return not self.lt(other) - def int_ge(self, other): - return not self.int_lt(other) + def int_ge(self, iother): + return not self.int_lt(iother) @jit.elidable def hash(self): @@ -634,20 +634,20 @@ return result @jit.elidable - def int_add(self, other): - if not int_in_valid_range(other): + def int_add(self, iother): + if not int_in_valid_range(iother): # Fallback to long. - return self.add(rbigint.fromint(other)) + return self.add(rbigint.fromint(iother)) elif self.sign == 0: - return rbigint.fromint(other) - elif other == 0: + return rbigint.fromint(iother) + elif iother == 0: return self - sign = intsign(other) + sign = intsign(iother) if self.sign == sign: - result = _x_int_add(self, other) + result = _x_int_add(self, iother) else: - result = _x_int_sub(self, other) + result = _x_int_sub(self, iother) result.sign *= -1 result.sign *= sign return result @@ -666,96 +666,94 @@ return result @jit.elidable - def int_sub(self, other): - if not int_in_valid_range(other): + def int_sub(self, iother): + if not int_in_valid_range(iother): # Fallback to long. - return self.sub(rbigint.fromint(other)) - elif other == 0: + return self.sub(rbigint.fromint(iother)) + elif iother == 0: return self elif self.sign == 0: - return rbigint.fromint(-other) - elif self.sign == intsign(other): - result = _x_int_sub(self, other) + return rbigint.fromint(-iother) + elif self.sign == intsign(iother): + result = _x_int_sub(self, iother) else: - result = _x_int_add(self, other) + result = _x_int_add(self, iother) result.sign *= self.sign return result @jit.elidable - def mul(self, b): - asize = self.numdigits() - bsize = b.numdigits() - - a = self - - if asize > bsize: - a, b, asize, bsize = b, a, bsize, asize - - if a.sign == 0 or b.sign == 0: + def mul(self, other): + selfsize = self.numdigits() + othersize = other.numdigits() + + if selfsize > othersize: + self, other, selfsize, othersize = other, self, othersize, selfsize + + if self.sign == 0 or other.sign == 0: return NULLRBIGINT - if asize == 1: - if a._digits[0] == ONEDIGIT: - return rbigint(b._digits[:bsize], a.sign * b.sign, bsize) - elif bsize == 1: - res = b.uwidedigit(0) * a.udigit(0) + if selfsize == 1: + if self._digits[0] == ONEDIGIT: + return rbigint(other._digits[:othersize], self.sign * other.sign, othersize) + elif othersize == 1: + res = other.uwidedigit(0) * self.udigit(0) carry = res >> SHIFT if carry: - return rbigint([_store_digit(res & MASK), _store_digit(carry)], a.sign * b.sign, 2) + return rbigint([_store_digit(res & MASK), _store_digit(carry)], self.sign * other.sign, 2) else: - return rbigint([_store_digit(res & MASK)], a.sign * b.sign, 1) - - result = _x_mul(a, b, a.digit(0)) + return rbigint([_store_digit(res & MASK)], self.sign * other.sign, 1) + + result = _x_mul(self, other, self.digit(0)) elif USE_KARATSUBA: - if a is b: + if self is other: i = KARATSUBA_SQUARE_CUTOFF else: i = KARATSUBA_CUTOFF - if asize <= i: - result = _x_mul(a, b) - """elif 2 * asize <= bsize: - result = _k_lopsided_mul(a, b)""" + if selfsize <= i: + result = _x_mul(self, other) + """elif 2 * selfsize <= othersize: + result = _k_lopsided_mul(self, other)""" else: - result = _k_mul(a, b) + result = _k_mul(self, other) else: - result = _x_mul(a, b) - - result.sign = a.sign * b.sign + result = _x_mul(self, other) + + result.sign = self.sign * other.sign return result @jit.elidable - def int_mul(self, b): - if not int_in_valid_range(b): + def int_mul(self, iother): + if not int_in_valid_range(iother): # Fallback to long. - return self.mul(rbigint.fromint(b)) - - if self.sign == 0 or b == 0: + return self.mul(rbigint.fromint(iother)) + + if self.sign == 0 or iother == 0: return NULLRBIGINT asize = self.numdigits() - digit = abs(b) - - bsign = intsign(b) + digit = abs(iother) + + othersign = intsign(iother) if digit == 1: - if bsign == 1: + if othersign == 1: return self - return rbigint(self._digits[:asize], self.sign * bsign, asize) + return rbigint(self._digits[:asize], self.sign * othersign, asize) elif asize == 1: udigit = r_uint(digit) res = self.uwidedigit(0) * udigit carry = res >> SHIFT if carry: - return rbigint([_store_digit(res & MASK), _store_digit(carry)], self.sign * bsign, 2) + return rbigint([_store_digit(res & MASK), _store_digit(carry)], self.sign * othersign, 2) else: - return rbigint([_store_digit(res & MASK)], self.sign * bsign, 1) + return rbigint([_store_digit(res & MASK)], self.sign * othersign, 1) elif digit & (digit - 1) == 0: result = self.lqshift(ptwotable[digit]) else: result = _muladd1(self, digit) - result.sign = self.sign * bsign + result.sign = self.sign * othersign return result @jit.elidable @@ -782,18 +780,18 @@ return self.floordiv(other) @jit.elidable - def int_floordiv(self, b): - if not int_in_valid_range(b): + def int_floordiv(self, iother): + if not int_in_valid_range(iother): # Fallback to long. - return self.floordiv(rbigint.fromint(b)) - - if b == 0: + return self.floordiv(rbigint.fromint(iother)) + + if iother == 0: raise ZeroDivisionError("long division by zero") - digit = abs(b) + digit = abs(iother) assert digit > 0 - if self.sign == 1 and b > 0: + if self.sign == 1 and iother > 0: if digit == 1: return self elif digit & (digit - 1) == 0: @@ -801,16 +799,16 @@ div, mod = _divrem1(self, digit) - if mod != 0 and self.sign * intsign(b) == -1: + if mod != 0 and self.sign * intsign(iother) == -1: if div.sign == 0: return ONENEGATIVERBIGINT div = div.int_add(1) - div.sign = self.sign * intsign(b) + div.sign = self.sign * intsign(iother) div._normalize() return div - def int_div(self, other): - return self.int_floordiv(other) + def int_div(self, iother): + return self.int_floordiv(iother) @jit.elidable def mod(self, other): @@ -830,24 +828,24 @@ return mod @jit.elidable - def int_mod(self, other): - if other == 0: + def int_mod(self, iother): + if iother == 0: raise ZeroDivisionError("long division or modulo by zero") if self.sign == 0: return NULLRBIGINT - elif not int_in_valid_range(other): + elif not int_in_valid_range(iother): # Fallback to long. - return self.mod(rbigint.fromint(other)) + return self.mod(rbigint.fromint(iother)) if 1: # preserve indentation to preserve history - digit = abs(other) + digit = abs(iother) if digit == 1: return NULLRBIGINT elif digit == 2: modm = self.digit(0) & 1 if modm: - return ONENEGATIVERBIGINT if other < 0 else ONERBIGINT + return ONENEGATIVERBIGINT if iother < 0 else ONERBIGINT return NULLRBIGINT elif digit & (digit - 1) == 0: mod = self.int_and_(digit - 1) @@ -868,12 +866,12 @@ return NULLRBIGINT mod = rbigint([rem], -1 if self.sign < 0 else 1, 1) - if mod.sign * intsign(other) == -1: - mod = mod.int_add(other) + if mod.sign * intsign(iother) == -1: + mod = mod.int_add(iother) return mod @jit.elidable - def divmod(v, w): + def divmod(self, other): """ The / and % operators are now defined in terms of divmod(). The expression a mod b has the value a - b*floor(a/b). @@ -890,39 +888,39 @@ have different signs. We then subtract one from the 'div' part of the outcome to keep the invariant intact. """ - div, mod = _divrem(v, w) - if mod.sign * w.sign == -1: - mod = mod.add(w) + div, mod = _divrem(self, other) + if mod.sign * other.sign == -1: + mod = mod.add(other) if div.sign == 0: return ONENEGATIVERBIGINT, mod div = div.int_sub(1) return div, mod @jit.elidable - def int_divmod(v, w): + def int_divmod(self, iother): """ Divmod with int """ - if w == 0: + if iother == 0: raise ZeroDivisionError("long division or modulo by zero") - wsign = intsign(w) - if not int_in_valid_range(w) or (wsign == -1 and v.sign != wsign): + wsign = intsign(iother) + if not int_in_valid_range(iother) or (wsign == -1 and self.sign != wsign): # Just fallback. - return v.divmod(rbigint.fromint(w)) - - digit = abs(w) + return self.divmod(rbigint.fromint(iother)) + + digit = abs(iother) assert digit > 0 - div, mod = _divrem1(v, digit) + div, mod = _divrem1(self, digit) # _divrem1 doesn't fix the sign if div.size == 1 and div._digits[0] == NULLDIGIT: div.sign = 0 else: - div.sign = v.sign * wsign - if v.sign < 0: + div.sign = self.sign * wsign + if self.sign < 0: mod = -mod - if mod and v.sign * wsign == -1: - mod += w + if mod and self.sign * wsign == -1: + mod += iother if div.sign == 0: div = ONENEGATIVERBIGINT else: @@ -931,37 +929,37 @@ return div, mod @jit.elidable - def pow(a, b, c=None): + def pow(self, other, modulus=None): negativeOutput = False # if x<0 return negative output # 5-ary values. If the exponent is large enough, table is - # precomputed so that table[i] == a**i % c for i in range(32). + # precomputed so that table[i] == self**i % modulus for i in range(32). # python translation: the table is computed when needed. - if b.sign < 0: # if exponent is negative - if c is not None: + if other.sign < 0: # if exponent is negative + if modulus is not None: raise TypeError( "pow() 2nd argument " "cannot be negative when 3rd argument specified") # XXX failed to implement raise ValueError("bigint pow() too negative") - size_b = UDIGIT_TYPE(b.numdigits()) - - if c is not None: - if c.sign == 0: + size_b = UDIGIT_TYPE(other.numdigits()) + + if modulus is not None: + if modulus.sign == 0: raise ValueError("pow() 3rd argument cannot be 0") # if modulus < 0: # negativeOutput = True # modulus = -modulus - if c.sign < 0: + if modulus.sign < 0: negativeOutput = True - c = c.neg() + modulus = modulus.neg() # if modulus == 1: # return 0 - if c.numdigits() == 1 and c._digits[0] == ONEDIGIT: + if modulus.numdigits() == 1 and modulus._digits[0] == ONEDIGIT: return NULLRBIGINT # Reduce base by modulus in some cases: @@ -973,30 +971,30 @@ # base % modulus instead. # We could _always_ do this reduction, but mod() isn't cheap, # so we only do it when it buys something. - if a.sign < 0 or a.numdigits() > c.numdigits(): - a = a.mod(c) - elif b.sign == 0: + if self.sign < 0 or self.numdigits() > modulus.numdigits(): + self = self.mod(modulus) + elif other.sign == 0: return ONERBIGINT - elif a.sign == 0: + elif self.sign == 0: return NULLRBIGINT elif size_b == 1: - if b._digits[0] == ONEDIGIT: - return a - elif a.numdigits() == 1 and c is None: - adigit = a.digit(0) - digit = b.digit(0) + if other._digits[0] == ONEDIGIT: + return self + elif self.numdigits() == 1 and modulus is None: + adigit = self.digit(0) + digit = other.digit(0) if adigit == 1: - if a.sign == -1 and digit % 2: + if self.sign == -1 and digit % 2: return ONENEGATIVERBIGINT return ONERBIGINT elif adigit & (adigit - 1) == 0: - ret = a.lshift(((digit-1)*(ptwotable[adigit]-1)) + digit-1) - if a.sign == -1 and not digit % 2: + ret = self.lshift(((digit-1)*(ptwotable[adigit]-1)) + digit-1) + if self.sign == -1 and not digit % 2: ret.sign = 1 return ret - # At this point a, b, and c are guaranteed non-negative UNLESS - # c is NULL, in which case a may be negative. */ + # At this point self, other, and modulus are guaranteed non-negative UNLESS + # modulus is NULL, in which case self may be negative. */ z = ONERBIGINT @@ -1008,26 +1006,26 @@ while size_b > 0: size_b -= 1 - bi = b.digit(size_b) + bi = other.digit(size_b) j = 1 << (SHIFT-1) while j != 0: - z = _help_mult(z, z, c) + z = _help_mult(z, z, modulus) if bi & j: - z = _help_mult(z, a, c) + z = _help_mult(z, self, modulus) j >>= 1 else: # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82) - # This is only useful in the case where c != None. + # This is only useful in the case where modulus != None. # z still holds 1L table = [z] * 32 table[0] = z for i in range(1, 32): - table[i] = _help_mult(table[i-1], a, c) + table[i] = _help_mult(table[i-1], self, modulus) # Note that here SHIFT is not a multiple of 5. The difficulty - # is to extract 5 bits at a time from 'b', starting from the + # is to extract 5 bits at a time from 'other', starting from the # most significant digits, so that at the end of the algorithm # it falls exactly to zero. # m = max number of bits = i * SHIFT @@ -1046,59 +1044,59 @@ index = (accum >> j) & 0x1f else: # 'accum' does not have enough digit. - # must get the next digit from 'b' in order to complete + # must get the next digit from 'other' in order to complete if size_b == 0: break # Done size_b -= 1 assert size_b >= 0 - bi = b.udigit(size_b) + bi = other.udigit(size_b) index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f accum = bi j += SHIFT # for k in range(5): - z = _help_mult(z, z, c) + z = _help_mult(z, z, modulus) if index: - z = _help_mult(z, table[index], c) + z = _help_mult(z, table[index], modulus) # assert j == -5 if negativeOutput and z.sign != 0: - z = z.sub(c) + z = z.sub(modulus) return z @jit.elidable - def int_pow(a, b, c=None): + def int_pow(self, iother, modulus=None): negativeOutput = False # if x<0 return negative output # 5-ary values. If the exponent is large enough, table is - # precomputed so that table[i] == a**i % c for i in range(32). + # precomputed so that table[i] == self**i % modulus for i in range(32). # python translation: the table is computed when needed. - if b < 0: # if exponent is negative - if c is not None: + if iother < 0: # if exponent is negative + if modulus is not None: raise TypeError( "pow() 2nd argument " "cannot be negative when 3rd argument specified") # XXX failed to implement raise ValueError("bigint pow() too negative") - assert b >= 0 - if c is not None: - if c.sign == 0: + assert iother >= 0 + if modulus is not None: + if modulus.sign == 0: raise ValueError("pow() 3rd argument cannot be 0") # if modulus < 0: # negativeOutput = True # modulus = -modulus - if c.sign < 0: + if modulus.sign < 0: negativeOutput = True - c = c.neg() + modulus = modulus.neg() # if modulus == 1: # return 0 - if c.numdigits() == 1 and c._digits[0] == ONEDIGIT: + if modulus.numdigits() == 1 and modulus._digits[0] == ONEDIGIT: return NULLRBIGINT # Reduce base by modulus in some cases: @@ -1110,45 +1108,45 @@ # base % modulus instead. # We could _always_ do this reduction, but mod() isn't cheap, # so we only do it when it buys something. - if a.sign < 0 or a.numdigits() > c.numdigits(): - a = a.mod(c) - elif b == 0: + if self.sign < 0 or self.numdigits() > modulus.numdigits(): + self = self.mod(modulus) + elif iother == 0: return ONERBIGINT - elif a.sign == 0: + elif self.sign == 0: return NULLRBIGINT - elif b == 1: - return a - elif a.numdigits() == 1: - adigit = a.digit(0) + elif iother == 1: + return self + elif self.numdigits() == 1: + adigit = self.digit(0) if adigit == 1: - if a.sign == -1 and b % 2: + if self.sign == -1 and iother % 2: return ONENEGATIVERBIGINT return ONERBIGINT elif adigit & (adigit - 1) == 0: - ret = a.lshift(((b-1)*(ptwotable[adigit]-1)) + b-1) - if a.sign == -1 and not b % 2: + ret = self.lshift(((iother-1)*(ptwotable[adigit]-1)) + iother-1) + if self.sign == -1 and not iother % 2: ret.sign = 1 return ret - # At this point a, b, and c are guaranteed non-negative UNLESS - # c is NULL, in which case a may be negative. */ + # At this point self, iother, and modulus are guaranteed non-negative UNLESS + # modulus is NULL, in which case self may be negative. */ z = ONERBIGINT # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result) - # into helper function result = _help_mult(x, y, c) + # into helper function result = _help_mult(x, y, modulus) # Left-to-right binary exponentiation (HAC Algorithm 14.79) # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf j = 1 << (SHIFT-1) while j != 0: - z = _help_mult(z, z, c) - if b & j: - z = _help_mult(z, a, c) + z = _help_mult(z, z, modulus) + if iother & j: + z = _help_mult(z, self, modulus) j >>= 1 if negativeOutput and z.sign != 0: - z = z.sub(c) + z = z.sub(modulus) return z @jit.elidable @@ -1349,24 +1347,24 @@ return _bitwise(self, '&', other) @jit.elidable - def int_and_(self, other): - return _int_bitwise(self, '&', other) + def int_and_(self, iother): + return _int_bitwise(self, '&', iother) @jit.elidable def xor(self, other): return _bitwise(self, '^', other) @jit.elidable - def int_xor(self, other): - return _int_bitwise(self, '^', other) + def int_xor(self, iother): + return _int_bitwise(self, '^', iother) @jit.elidable def or_(self, other): return _bitwise(self, '|', other) @jit.elidable - def int_or_(self, other): - return _int_bitwise(self, '|', other) + def int_or_(self, iother): + return _int_bitwise(self, '|', iother) @jit.elidable def oct(self): @@ -2886,7 +2884,7 @@ elif s[p] == '+': p += 1 - a = rbigint() + a = NULLRBIGINT tens = 1 dig = 0 ord0 = ord('0') @@ -2907,7 +2905,7 @@ base = parser.base if (base & (base - 1)) == 0 and base >= 2: return parse_string_from_binary_base(parser) - a = rbigint() + a = NULLRBIGINT digitmax = BASE_MAX[base] tens, dig = 1, 0 while True: From pypy.commits at gmail.com Thu Dec 13 07:18:43 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 04:18:43 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: cover some more cases Message-ID: <5c124e23.1c69fb81.57fd5.8c55@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95470:0024322f473e Date: 2018-12-13 11:16 +0100 http://bitbucket.org/pypy/pypy/changeset/0024322f473e/ Log: cover some more cases diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -639,6 +639,9 @@ res3 = f1.abs_rshift_and_mask(r_ulonglong(y), mask) assert res3 == (abs(x) >> y) & mask + # test special optimization case in rshift: + assert rbigint.fromlong(-(1 << 100)).rshift(5).tolong() == -(1 << 100) >> 5 + def test_qshift(self): for x in range(10): for y in range(1, 161, 16): @@ -916,6 +919,7 @@ assert rem1.tolong() == _rem assert div.tolong() == _div assert rem.tolong() == _rem + py.test.raises(ZeroDivisionError, rbigint.fromlong(x).int_divmod, 0) # testing Karatsuba stuff def test__v_iadd(self): @@ -1157,3 +1161,34 @@ assert rbigint.fromint(x).hash() == x # hash of negative machine-sized integers assert rbigint.fromint(-x-1).hash() == -x-1 + + @given(longs) + def test_abs(self, x): + assert rbigint.fromlong(x).abs().tolong() == abs(x) + + @given(longs, longs) + def test_truediv(self, a, b): + ra = rbigint.fromlong(a) + rb = rbigint.fromlong(b) + if not b: + pytest.raises(ZeroDivisionError, ra.truediv, rb) + else: + assert ra.truediv(rb) == a / b + + @given(longs, longs) + def test_bitwise(self, x, y): + lx = rbigint.fromlong(x) + ly = rbigint.fromlong(y) + for mod in "xor and_ or_".split(): + res1 = getattr(lx, mod)(ly).tolong() + res2 = getattr(operator, mod)(x, y) + assert res1 == res2 + + @given(longs, ints) + def test_int_bitwise(self, x, y): + lx = rbigint.fromlong(x) + for mod in "xor and_ or_".split(): + res1 = getattr(lx, 'int_' + mod)(y).tolong() + res2 = getattr(operator, mod)(x, y) + assert res1 == res2 + From pypy.commits at gmail.com Thu Dec 13 07:18:45 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 04:18:45 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: use CPython's long formatting in debug-only __repr__ Message-ID: <5c124e25.1c69fb81.2d293.6c9e@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95471:5b9d394a43cd Date: 2018-12-13 13:08 +0100 http://bitbucket.org/pypy/pypy/changeset/5b9d394a43cd/ Log: use CPython's long formatting in debug-only __repr__ diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1439,7 +1439,7 @@ def __repr__(self): return "" % (self._digits, self.sign, self.numdigits(), len(self._digits), - self.str()) + self.tolong()) ONERBIGINT = rbigint([ONEDIGIT], 1, 1) ONENEGATIVERBIGINT = rbigint([ONEDIGIT], -1, 1) @@ -2178,7 +2178,7 @@ if d1 < b: return True return False - + # ______________ conversions to double _______________ def _AsScaledDouble(v): From pypy.commits at gmail.com Thu Dec 13 07:18:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 04:18:48 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: improve test coverage by adding an explicit test for a rare case in division Message-ID: <5c124e28.1c69fb81.46b9b.72fc@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95472:bab1896b4351 Date: 2018-12-13 13:09 +0100 http://bitbucket.org/pypy/pypy/changeset/bab1896b4351/ Log: improve test coverage by adding an explicit test for a rare case in division diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -900,6 +900,18 @@ assert rem.tolong() == _rem py.test.raises(ZeroDivisionError, rbigint.fromlong(x).divmod, rbigint.fromlong(0)) + # an explicit example for a very rare case in _x_divrem: + # "add w back if q was too large (this branch taken rarely)" + x = 2401064762424988628303678384283622960038813848808995811101817752058392725584695633 + y = 510439143470502793407446782273075179624699774495710665331026 + f1 = rbigint.fromlong(x) + f2 = rbigint.fromlong(y) + div, rem = f1.divmod(f2) + _div, _rem = divmod(x, y) + assert div.tolong() == _div + assert rem.tolong() == _rem + + def test_int_divmod(self): for x in long_vals: for y in int_vals + [-sys.maxint-1]: From pypy.commits at gmail.com Thu Dec 13 07:18:50 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 04:18:50 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: improve random testing of divmod, this finds a bug! Message-ID: <5c124e2a.1c69fb81.28ade.a319@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95473:6b10ba9fde9c Date: 2018-12-13 13:13 +0100 http://bitbucket.org/pypy/pypy/changeset/6b10ba9fde9c/ Log: improve random testing of divmod, this finds a bug! bad Stian, no cookie diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -16,12 +16,22 @@ from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_standalone import StandaloneTests -from hypothesis import given, strategies +from hypothesis import given, strategies, example longs = strategies.builds( long, strategies.integers()) ints = strategies.integers(-sys.maxint-1, sys.maxint) +def makelong(data): + numbits = data.draw(strategies.integers(1, 2000)) + r = data.draw(strategies.integers(0, 1 << numbits)) + if data.draw(strategies.booleans()): + return -r + return r + +biglongs = strategies.builds(makelong, strategies.data()) + + def gen_signs(l): for s in l: if s == 0: @@ -1125,7 +1135,6 @@ data = cbuilder.cmdexec('hi there') assert data == '[%d]\n[0, 1]\n' % sys.maxint - class TestHypothesis(object): @given(longs, longs, longs) def test_pow(self, x, y, z): @@ -1140,23 +1149,35 @@ v = f1.pow(f2, f3) assert v.tolong() == res - @given(longs, longs, longs) - def test_divmod(self, x, y, z): + @given(biglongs, biglongs) + @example(510439143470502793407446782273075179618477362188870662225920, + 108089693021945158982483698831267549521) + def test_divmod(self, x, y): + if x < y: + x, y = y, x + f1 = rbigint.fromlong(x) f2 = rbigint.fromlong(y) - f3 = rbigint.fromlong(z) try: - res = divmod(x, y, z) + res = divmod(x, y) except Exception as e: - pytest.raises(type(e), f1.divmod, f2, f3) - if isinstance(int(y), int): - pytest.raises(type(e), f1.int_divmod, f2, f3) + pytest.raises(type(e), f1.divmod, f2) else: - a, b = f1.divmod(f2, f3) - assert a.tolong(), b.tolong() == res - if isinstance(int(y), int): - a, b = f1.int_divmod(f2, f3) - assert a.tolong(), b.tolong() == res + print x, y + a, b = f1.divmod(f2) + assert (a.tolong(), b.tolong()) == res + + @given(biglongs, ints) + def test_int_divmod(self, x, iy): + f1 = rbigint.fromlong(x) + try: + res = divmod(x, iy) + except Exception as e: + pytest.raises(type(e), f1.int_divmod, iy) + else: + print x, iy + a, b = f1.int_divmod(iy) + assert (a.tolong(), b.tolong()) == res @given(longs) def test_hash(self, x): From pypy.commits at gmail.com Thu Dec 13 10:57:38 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 13 Dec 2018 07:57:38 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: hg merge default Message-ID: <5c128172.1c69fb81.c9108.9b82@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95474:9716697bf2dc Date: 2018-12-13 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/9716697bf2dc/ Log: hg merge default diff too long, truncating to 2000 out of 3781 lines diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -182,6 +182,8 @@ module = category[:i] klass = category[i+1:] try: + if not module: + raise ImportError # instead of the ValueError we'd get m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -26,6 +26,12 @@ from pyrepl.fancy_termios import tcgetattr, tcsetattr from pyrepl.console import Console, Event from pyrepl import unix_eventqueue +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -70,8 +76,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -416,6 +422,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError), err: diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -4,7 +4,7 @@ .. contents:: This document gives an overview of the goals and architecture of PyPy. If you're -interested in :ref:`using PyPy ` or :ref:`hacking on it `, +interested in :ref:`using PyPy ` or hacking on it, have a look at our :ref:`getting started ` section. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -192,10 +192,10 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('temp_index', 'PyPy.tex', u'PyPy Documentation', - u'The PyPy Project', 'manual'), -] +#latex_documents = [ +# ('temp_index', 'PyPy.tex', u'PyPy Documentation', +# u'The PyPy Project', 'manual'), +#] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -212,7 +212,7 @@ #latex_appendices = [] # If false, no module index is generated. -latex_use_modindex = False +#latex_use_modindex = False # Example configuration for intersphinx: refer to the Python standard library. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -29,6 +29,7 @@ :maxdepth: 1 introduction + architecture install build windows @@ -59,7 +60,6 @@ :maxdepth: 2 contributing - architecture configuration project-ideas project-documentation diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst --- a/pypy/doc/project-documentation.rst +++ b/pypy/doc/project-documentation.rst @@ -28,7 +28,6 @@ .. toctree:: :hidden: - architecture coding-guide sprint-reports extradoc diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -111,9 +111,12 @@ -------------- Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. -Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy -test suite. We could use help analyzing the failures and fixing them either -as patches to upstream NumPy, or as fixes to PyPy. +Release PyPy2.7-v6.0 still fails about 10 of the ~6000 test in the NumPy +test suite. We need to improve our ctypes structure -> memoryview conversions_, +and to refactor the way `NumPy adds docstrings`_. + +.. _conversions: https://bitbucket.org/pypy/pypy/issues/2930 +.. _`NumPy adds docstrings`: https://github.com/numpy/numpy/issues/10167 We also are looking for help in how to hijack NumPy dtype conversion and ufunc calls to allow the JIT to make them fast, using our internal _numpypy @@ -165,8 +168,33 @@ Or maybe not. We can also play around with the idea of using a single representation: as a byte string in utf-8. (This idea needs some extra logic -for efficient indexing, like a cache.) +for efficient indexing, like a cache.) Work has begun on the ``unicode-utf`` +and ``unicode-utf8-py3`` branches. More is needed, for instance there are +SIMD optimizations that are not yet used. +Convert RPython to Python3 +-------------------------- + +The world is moving on, we should too. + +Improve performance +------------------- + +* Make uninlined Python-level calls faster +* Switch to a `sea-of-nodes`_ IR, or a `Lua-Jit`_-like IR which iterates on + on the sea-of-nodes approach +* Use real register-allocation +* Improve instruction selection / scheduling +* Create a hybrid tracing/method JIT + +.. _`sea-of-nodes`: https://darksi.de/d.sea-of-nodes/ +.. _`Lua-JIT`: http://wiki.luajit.org/SSA-IR-2.0 + +Improve warmup +-------------- +* Interpreter speed-ups +* Optimize while tracing +* Cache information between runs Translation Toolchain --------------------- @@ -234,6 +262,27 @@ .. _runner: http://speed.pypy.org .. _`CPython site`: https://speed.python.org/ + +Interfacing with C +------------------ + +While we could make ``cpyext`` faster_, we would also like to explore other +ideas. It seems cffi is only appropriate for small to medium-sized extensions, +and it is hard to imagine NumPy abandoning the C-API. Here are a few ideas: +* Extend Cython to have a backend that can be understood by the JIT +* Collaborate with C-extension authors to ensure full PyPy support (see below) +* Put PyPy compatible packages on PyPI and in conda + + +.. _faster: https://morepypy.blogspot.com/2018/09#next-steps + +Support more platforms +---------------------- + +We have a plan for a `Windows 64`_ port. + +.. _`Windows 64`: windows.html#what-is-missing-for-a-full-64-bit-translation + ====================================== Make more python modules pypy-friendly ====================================== diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,3 +39,20 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -609,8 +609,14 @@ warnoptions.extend(pythonwarnings.split(',')) if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -977,6 +977,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -109,6 +109,7 @@ '_promote' : 'interp_magic._promote', 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -211,3 +211,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,23 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -383,3 +392,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,15 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -644,6 +644,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', @@ -1180,6 +1181,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1726,6 +1731,12 @@ w_mod = state.fixup_extension(name, path) return w_mod +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -47,6 +47,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -261,6 +261,7 @@ if w_obj is not None: if w_obj is not w_marker_deallocating: return w_obj + type_name = rffi.charp2str(cts.cast('char*', ref.c_ob_type.c_tp_name)) fatalerror( "*** Invalid usage of a dying CPython object ***\n" "\n" @@ -275,7 +276,8 @@ "freed, making that reference point to garbage.\n" ">>> PyPy could contain some workaround to still work if\n" "you are lucky, but it is not done so far; better fix the bug in\n" - "the CPython extension.") + "the CPython extension.\n" + ">>> This object is of type '%s'" % (type_name,)) # This reference is not yet a real interpreter object. # Realize it. diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyInt_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -421,6 +421,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -366,10 +366,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -398,28 +402,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -8,6 +8,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -139,9 +141,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -149,9 +151,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -179,9 +181,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -189,9 +191,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -307,10 +309,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -334,7 +340,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -355,7 +361,7 @@ "major_is_done", cls=W_GcCollectStepStats, wrapfn="newbool"), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -366,7 +372,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -211,9 +211,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -576,6 +576,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest From pypy.commits at gmail.com Thu Dec 13 12:11:22 2018 From: pypy.commits at gmail.com (Stian Andreassen) Date: Thu, 13 Dec 2018 09:11:22 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: Fix _x_divrem (thanks Carl Friedrich Bolz-Tereick for the test) Message-ID: <5c1292ba.1c69fb81.efb2e.a324@mx.google.com> Author: Stian Andreassen Branch: math-improvements Changeset: r95475:918c7b88d892 Date: 2018-12-13 18:10 +0100 http://bitbucket.org/pypy/pypy/changeset/918c7b88d892/ Log: Fix _x_divrem (thanks Carl Friedrich Bolz-Tereick for the test) diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -2073,12 +2073,15 @@ if j >= size_v: vtop = 0 else: - vtop = v.widedigit(j) << SHIFT - - vv = vtop | v.digit(abs(j-1)) + vtop = v.widedigit(j) + assert vtop <= wm1 + + vv = (vtop << SHIFT) | v.widedigit(abs(j-1)) + # Hints to make division just as fast as doing it unsigned. But avoids casting to get correct results. assert vv >= 0 assert wm1 >= 1 + q = vv / wm1 r = vv % wm1 # This seems to be slightly faster on widen digits than vv - wm1 * q. vj2 = v.digit(abs(j-2)) From pypy.commits at gmail.com Thu Dec 13 13:56:53 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 10:56:53 -0800 (PST) Subject: [pypy-commit] pypy default: cherry-pick some of the test from math-improvements to default Message-ID: <5c12ab75.1c69fb81.fc6ac.4c78@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95476:a838af77f2af Date: 2018-12-13 13:25 +0100 http://bitbucket.org/pypy/pypy/changeset/a838af77f2af/ Log: cherry-pick some of the test from math-improvements to default diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -671,6 +671,13 @@ assert type(x) is int assert str(x) == "0" + def test_binop_overflow(self): + x = int(2) + assert x.__lshift__(128) == 680564733841876926926749214863536422912L + + def test_rbinop_overflow(self): + x = int(321) + assert x.__rlshift__(333) == 1422567365923326114875084456308921708325401211889530744784729710809598337369906606315292749899759616L class AppTestIntShortcut(AppTestInt): spaceconfig = {"objspace.std.intshortcut": True} diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -2,7 +2,6 @@ from pypy.objspace.std import longobject as lobj from rpython.rlib.rbigint import rbigint - class TestW_LongObject: def test_bigint_w(self): space = self.space @@ -52,8 +51,9 @@ assert x + 2 + 3L + True == -14L def test_sub(self): - x = 58543L - assert int(x - 12332L) == 58543 - 12332 + assert int(58543L - 12332L) == 58543 - 12332 + assert int(58543L - 12332) == 58543 - 12332 + assert int(58543 - 12332L) == 58543 - 12332 x = 237123838281233L assert x * 12 == x * 12L @@ -70,6 +70,23 @@ a = x // 10000000L assert a == 3L + def test_int_floordiv(self): + import sys + + x = 3000L + a = x // 1000 + assert a == 3L + + x = 3000L + a = x // -1000 + assert a == -3L + + x = 3000L + raises(ZeroDivisionError, "x // 0") + + n = sys.maxint+1 + assert n / int(-n) == -1L + def test_numerator_denominator(self): assert (1L).numerator == 1L assert (1L).denominator == 1L @@ -79,25 +96,27 @@ def test_compare(self): Z = 0 ZL = 0L + + assert Z == ZL + assert not (Z != ZL) + assert ZL == Z + assert not (ZL != Z) + assert Z <= ZL + assert not (Z < ZL) + assert ZL <= ZL + assert not (ZL < ZL) + for BIG in (1L, 1L << 62, 1L << 9999): - assert Z == ZL - assert not (Z != ZL) - assert ZL == Z - assert not (ZL != Z) assert not (Z == BIG) assert Z != BIG assert not (BIG == Z) assert BIG != Z assert not (ZL == BIG) assert ZL != BIG - assert Z <= ZL - assert not (Z < ZL) assert Z <= BIG assert Z < BIG assert not (BIG <= Z) assert not (BIG < Z) - assert ZL <= ZL - assert not (ZL < ZL) assert ZL <= BIG assert ZL < BIG assert not (BIG <= ZL) @@ -171,10 +190,38 @@ assert type(-long2(0)) is long assert type(long2(5) // 1) is long + def test_shift(self): + assert 65l >> 2l == 16l + assert 65l >> 2 == 16l + assert 65 >> 2l == 16l + assert 65l << 2l == 65l * 4 + assert 65l << 2 == 65l * 4 + assert 65 << 2l == 65l * 4 + raises(ValueError, "1L << -1L") + raises(ValueError, "1L << -1") + raises(OverflowError, "1L << (2 ** 100)") + raises(ValueError, "1L >> -1L") + raises(ValueError, "1L >> -1") + raises(OverflowError, "1L >> (2 ** 100)") + def test_pow(self): x = 0L assert pow(x, 0L, 1L) == 0L assert pow(-1L, -1L) == -1.0 + assert pow(2 ** 68, 0.5) == 2.0 ** 34 + assert pow(2 ** 68, 2) == 2 ** 136 + raises(TypeError, pow, 2l, -1, 3) + raises(ValueError, pow, 2l, 5, 0) + + # some rpow tests + assert pow(0, 0L, 1L) == 0L + assert pow(-1, -1L) == -1.0 + + def test_int_pow(self): + x = 2L + assert pow(x, 2) == 4L + assert pow(x, 2, 2) == 0L + assert pow(x, 2, 3L) == 1L def test_getnewargs(self): assert 0L .__getnewargs__() == (0L,) @@ -185,8 +232,8 @@ q, r = divmod(x, y) pab, pba = x*y, y*x assert pab == pba - assert q == x//y - assert r == x%y + assert q == x // y + assert r == x % y assert x == q*y + r if y > 0: assert 0 <= r < y @@ -196,6 +243,8 @@ for y in [-105566530L, -1L, 1L, 1034522340L]: print "checking division for %s, %s" % (x, y) check_division(x, y) + check_division(x, int(y)) + check_division(int(x), y) # special case from python tests: s1 = 33 s2 = 2 @@ -207,6 +256,16 @@ y = 0x9800FFC1L check_division(x, y) raises(ZeroDivisionError, "x // 0L") + raises(ZeroDivisionError, "x % 0L") + raises(ZeroDivisionError, divmod, x, 0L) + raises(ZeroDivisionError, "x // 0") + raises(ZeroDivisionError, "x % 0") + raises(ZeroDivisionError, divmod, x, 0) + + def test_int_divmod(self): + q, r = divmod(100L, 11) + assert q == 9L + assert r == 1L def test_format(self): assert repr(12345678901234567890) == '12345678901234567890L' @@ -321,6 +380,10 @@ return 42 assert long(A('abc')) == 42 + def test_long_errors(self): + raises(TypeError, long, 12, 12) + raises(ValueError, long, 'xxxxxx?', 12) + def test_conjugate(self): assert (7L).conjugate() == 7L assert (-7L).conjugate() == -7L @@ -386,3 +449,4 @@ n = "a" * size expected = (2 << (size * 4)) // 3 assert long(n, 16) == expected + From pypy.commits at gmail.com Thu Dec 13 13:56:55 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 10:56:55 -0800 (PST) Subject: [pypy-commit] pypy default: port the new tests from math-improvements to default Message-ID: <5c12ab77.1c69fb81.d50b9.3617@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95477:bbea6977d260 Date: 2018-12-13 19:53 +0100 http://bitbucket.org/pypy/pypy/changeset/bbea6977d260/ Log: port the new tests from math-improvements to default diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -6,6 +6,7 @@ from random import random, randint, sample, seed import py +import pytest from rpython.rlib import rbigint as lobj from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask @@ -15,7 +16,21 @@ from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_standalone import StandaloneTests -from hypothesis import given, strategies +from hypothesis import given, strategies, example + +longs = strategies.builds( + long, strategies.integers()) +ints = strategies.integers(-sys.maxint-1, sys.maxint) + +def makelong(data): + numbits = data.draw(strategies.integers(1, 2000)) + r = data.draw(strategies.integers(0, 1 << numbits)) + if data.draw(strategies.booleans()): + return -r + return r + +biglongs = strategies.builds(makelong, strategies.data()) + def gen_signs(l): for s in l: @@ -44,8 +59,8 @@ class TestRLong(object): def test_simple(self): - for op1 in [-2, -1, 0, 1, 2, 50]: - for op2 in [-2, -1, 0, 1, 2, 50]: + for op1 in [-2, -1, 0, 1, 2, 10, 50]: + for op2 in [-2, -1, 0, 1, 2, 10, 50]: rl_op1 = rbigint.fromint(op1) rl_op2 = rbigint.fromint(op2) for op in "add sub mul".split(): @@ -155,13 +170,25 @@ def test_pow(self): for op1 in gen_signs(long_vals_not_too_big): + rl_op1 = rbigint.fromlong(op1) for op2 in [0, 1, 2, 8, 9, 10, 11]: - rl_op1 = rbigint.fromlong(op1) rl_op2 = rbigint.fromint(op2) r1 = rl_op1.pow(rl_op2) r2 = op1 ** op2 assert r1.tolong() == r2 + for op3 in gen_signs([1, 2, 5, 1000, 12312312312312235659969696l]): + if not op3: + continue + print op1, op2, op3 + r3 = rl_op1.pow(rl_op2, rbigint.fromlong(op3)) + r4 = pow(op1, op2, op3) + assert r3.tolong() == r4 + + def test_pow_raises(self): + r1 = rbigint.fromint(2) + r0 = rbigint.fromint(0) + py.test.raises(ValueError, r1.pow, r1, r0) def test_touint(self): result = r_uint(sys.maxint + 42) @@ -503,20 +530,6 @@ f1, f2, f3 = [rbigint.fromlong(i) for i in (10L, 5L, 0L)] py.test.raises(ValueError, f1.pow, f2, f3) - # - MAX = 1E20 - x = long(random() * MAX) + 1 - y = long(random() * MAX) + 1 - z = long(random() * MAX) + 1 - f1 = rbigint.fromlong(x) - f2 = rbigint.fromlong(y) - f3 = rbigint.fromlong(z) - print f1 - print f2 - print f3 - v = f1.pow(f2, f3) - print '--->', v - assert v.tolong() == pow(x, y, z) def test_pow_lll_bug(self): two = rbigint.fromint(2) @@ -585,6 +598,24 @@ res3 = f1.abs_rshift_and_mask(r_ulonglong(y), mask) assert res3 == (abs(x) >> y) & mask + # test special optimization case in rshift: + assert rbigint.fromlong(-(1 << 100)).rshift(5).tolong() == -(1 << 100) >> 5 + + def test_qshift(self): + for x in range(10): + for y in range(1, 161, 16): + num = (x << y) + x + f1 = rbigint.fromlong(num) + nf1 = rbigint.fromlong(-num) + + for z in range(1, 31): + res1 = f1.lqshift(z).tolong() + res3 = nf1.lqshift(z).tolong() + + assert res1 == num << z + assert res3 == -num << z + + def test_from_list_n_bits(self): for x in ([3L ** 30L, 5L ** 20L, 7 ** 300] + [1L << i for i in range(130)] + @@ -674,19 +705,12 @@ def test_hash(self): for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, sys.maxint-3, sys.maxint-2, sys.maxint-1, sys.maxint, - ] + [randint(0, sys.maxint) for _ in range(100)]: + ]: # hash of machine-sized integers assert rbigint.fromint(i).hash() == i # hash of negative machine-sized integers assert rbigint.fromint(-i-1).hash() == -i-1 # - for i in range(200): - # hash of large integers: should be equal to the hash of the - # integer reduced modulo 2**64-1, to make decimal.py happy - x = randint(0, sys.maxint**5) - y = x % (2**64-1) - assert rbigint.fromlong(x).hash() == rbigint.fromlong(y).hash() - assert rbigint.fromlong(-x).hash() == rbigint.fromlong(-y).hash() def test_log(self): from rpython.rlib.rfloat import ulps_check @@ -756,6 +780,7 @@ f1 = rbigint.fromlong(x) f2 = y remainder = lobj._inplace_divrem1(f1, f1, f2) + f1._normalize() assert (f1.tolong(), remainder) == divmod(x, y) out = bigint([99, 99], 1) remainder = lobj._inplace_divrem1(out, out, 100) @@ -787,7 +812,7 @@ y += randint(1, 1 << 60) if y > x: x <<= 100 - + f1 = rbigint.fromlong(x) f2 = rbigint.fromlong(y) div, rem = lobj._x_divrem(f1, f2) @@ -827,6 +852,18 @@ assert rem.tolong() == _rem py.test.raises(ZeroDivisionError, rbigint.fromlong(x).divmod, rbigint.fromlong(0)) + # an explicit example for a very rare case in _x_divrem: + # "add w back if q was too large (this branch taken rarely)" + x = 2401064762424988628303678384283622960038813848808995811101817752058392725584695633 + y = 510439143470502793407446782273075179624699774495710665331026 + f1 = rbigint.fromlong(x) + f2 = rbigint.fromlong(y) + div, rem = f1.divmod(f2) + _div, _rem = divmod(x, y) + assert div.tolong() == _div + assert rem.tolong() == _rem + + # testing Karatsuba stuff def test__v_iadd(self): f1 = bigint([lobj.MASK] * 10, 1) @@ -858,14 +895,6 @@ ret = lobj._k_mul(f1, f2) assert ret.tolong() == f1.tolong() * f2.tolong() - def test__k_lopsided_mul(self): - digs_a = KARATSUBA_CUTOFF + 3 - digs_b = 3 * digs_a - f1 = bigint([lobj.MASK] * digs_a, 1) - f2 = bigint([lobj.MASK] * digs_b, 1) - ret = lobj._k_lopsided_mul(f1, f2) - assert ret.tolong() == f1.tolong() * f2.tolong() - def test_longlong(self): max = 1L << (r_longlong.BITS-1) f1 = rbigint.fromlong(max-1) # fits in r_longlong @@ -1026,3 +1055,73 @@ t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data == '[%d]\n[0, 1]\n' % sys.maxint + +class TestHypothesis(object): + @given(longs, longs, longs) + def test_pow(self, x, y, z): + f1 = rbigint.fromlong(x) + f2 = rbigint.fromlong(y) + f3 = rbigint.fromlong(z) + try: + res = pow(x, y, z) + except Exception as e: + pytest.raises(type(e), f1.pow, f2, f3) + else: + v = f1.pow(f2, f3) + assert v.tolong() == res + + @given(biglongs, biglongs) + @example(510439143470502793407446782273075179618477362188870662225920, + 108089693021945158982483698831267549521) + def test_divmod(self, x, y): + if x < y: + x, y = y, x + + f1 = rbigint.fromlong(x) + f2 = rbigint.fromlong(y) + try: + res = divmod(x, y) + except Exception as e: + pytest.raises(type(e), f1.divmod, f2) + else: + print x, y + a, b = f1.divmod(f2) + assert (a.tolong(), b.tolong()) == res + + @given(longs) + def test_hash(self, x): + # hash of large integers: should be equal to the hash of the + # integer reduced modulo 2**64-1, to make decimal.py happy + x = randint(0, sys.maxint**5) + y = x % (2**64-1) + assert rbigint.fromlong(x).hash() == rbigint.fromlong(y).hash() + assert rbigint.fromlong(-x).hash() == rbigint.fromlong(-y).hash() + + @given(ints) + def test_hash_int(self, x): + # hash of machine-sized integers + assert rbigint.fromint(x).hash() == x + # hash of negative machine-sized integers + assert rbigint.fromint(-x-1).hash() == -x-1 + + @given(longs) + def test_abs(self, x): + assert rbigint.fromlong(x).abs().tolong() == abs(x) + + @given(longs, longs) + def test_truediv(self, a, b): + ra = rbigint.fromlong(a) + rb = rbigint.fromlong(b) + if not b: + pytest.raises(ZeroDivisionError, ra.truediv, rb) + else: + assert ra.truediv(rb) == a / b + + @given(longs, longs) + def test_bitwise(self, x, y): + lx = rbigint.fromlong(x) + ly = rbigint.fromlong(y) + for mod in "xor and_ or_".split(): + res1 = getattr(lx, mod)(ly).tolong() + res2 = getattr(operator, mod)(x, y) + assert res1 == res2 From pypy.commits at gmail.com Thu Dec 13 13:57:03 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 10:57:03 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge haeds Message-ID: <5c12ab7f.1c69fb81.fc7cb.2f47@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95479:51f9962bc56f Date: 2018-12-13 19:56 +0100 http://bitbucket.org/pypy/pypy/changeset/51f9962bc56f/ Log: merge haeds diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -2073,12 +2073,15 @@ if j >= size_v: vtop = 0 else: - vtop = v.widedigit(j) << SHIFT - - vv = vtop | v.digit(abs(j-1)) + vtop = v.widedigit(j) + assert vtop <= wm1 + + vv = (vtop << SHIFT) | v.widedigit(abs(j-1)) + # Hints to make division just as fast as doing it unsigned. But avoids casting to get correct results. assert vv >= 0 assert wm1 >= 1 + q = vv / wm1 r = vv % wm1 # This seems to be slightly faster on widen digits than vv - wm1 * q. vj2 = v.digit(abs(j-2)) From pypy.commits at gmail.com Thu Dec 13 13:57:01 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 13 Dec 2018 10:57:01 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge default Message-ID: <5c12ab7d.1c69fb81.eb088.366f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: math-improvements Changeset: r95478:5459ba014b0e Date: 2018-12-13 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/5459ba014b0e/ Log: merge default diff too long, truncating to 2000 out of 3510 lines diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -182,6 +182,8 @@ module = category[:i] klass = category[i+1:] try: + if not module: + raise ImportError # instead of the ValueError we'd get m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -26,6 +26,12 @@ from pyrepl.fancy_termios import tcgetattr, tcsetattr from pyrepl.console import Console, Event from pyrepl import unix_eventqueue +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -416,6 +422,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError), err: diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -4,7 +4,7 @@ .. contents:: This document gives an overview of the goals and architecture of PyPy. If you're -interested in :ref:`using PyPy ` or :ref:`hacking on it `, +interested in :ref:`using PyPy ` or hacking on it, have a look at our :ref:`getting started ` section. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -192,10 +192,10 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('temp_index', 'PyPy.tex', u'PyPy Documentation', - u'The PyPy Project', 'manual'), -] +#latex_documents = [ +# ('temp_index', 'PyPy.tex', u'PyPy Documentation', +# u'The PyPy Project', 'manual'), +#] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -212,7 +212,7 @@ #latex_appendices = [] # If false, no module index is generated. -latex_use_modindex = False +#latex_use_modindex = False # Example configuration for intersphinx: refer to the Python standard library. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -29,6 +29,7 @@ :maxdepth: 1 introduction + architecture install build windows @@ -59,7 +60,6 @@ :maxdepth: 2 contributing - architecture configuration project-ideas project-documentation diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst --- a/pypy/doc/project-documentation.rst +++ b/pypy/doc/project-documentation.rst @@ -28,7 +28,6 @@ .. toctree:: :hidden: - architecture coding-guide sprint-reports extradoc diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -111,9 +111,12 @@ -------------- Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. -Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy -test suite. We could use help analyzing the failures and fixing them either -as patches to upstream NumPy, or as fixes to PyPy. +Release PyPy2.7-v6.0 still fails about 10 of the ~6000 test in the NumPy +test suite. We need to improve our ctypes structure -> memoryview conversions_, +and to refactor the way `NumPy adds docstrings`_. + +.. _conversions: https://bitbucket.org/pypy/pypy/issues/2930 +.. _`NumPy adds docstrings`: https://github.com/numpy/numpy/issues/10167 We also are looking for help in how to hijack NumPy dtype conversion and ufunc calls to allow the JIT to make them fast, using our internal _numpypy @@ -165,8 +168,33 @@ Or maybe not. We can also play around with the idea of using a single representation: as a byte string in utf-8. (This idea needs some extra logic -for efficient indexing, like a cache.) +for efficient indexing, like a cache.) Work has begun on the ``unicode-utf`` +and ``unicode-utf8-py3`` branches. More is needed, for instance there are +SIMD optimizations that are not yet used. +Convert RPython to Python3 +-------------------------- + +The world is moving on, we should too. + +Improve performance +------------------- + +* Make uninlined Python-level calls faster +* Switch to a `sea-of-nodes`_ IR, or a `Lua-Jit`_-like IR which iterates on + on the sea-of-nodes approach +* Use real register-allocation +* Improve instruction selection / scheduling +* Create a hybrid tracing/method JIT + +.. _`sea-of-nodes`: https://darksi.de/d.sea-of-nodes/ +.. _`Lua-JIT`: http://wiki.luajit.org/SSA-IR-2.0 + +Improve warmup +-------------- +* Interpreter speed-ups +* Optimize while tracing +* Cache information between runs Translation Toolchain --------------------- @@ -234,6 +262,27 @@ .. _runner: http://speed.pypy.org .. _`CPython site`: https://speed.python.org/ + +Interfacing with C +------------------ + +While we could make ``cpyext`` faster_, we would also like to explore other +ideas. It seems cffi is only appropriate for small to medium-sized extensions, +and it is hard to imagine NumPy abandoning the C-API. Here are a few ideas: +* Extend Cython to have a backend that can be understood by the JIT +* Collaborate with C-extension authors to ensure full PyPy support (see below) +* Put PyPy compatible packages on PyPI and in conda + + +.. _faster: https://morepypy.blogspot.com/2018/09#next-steps + +Support more platforms +---------------------- + +We have a plan for a `Windows 64`_ port. + +.. _`Windows 64`: windows.html#what-is-missing-for-a-full-64-bit-translation + ====================================== Make more python modules pypy-friendly ====================================== diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,8 +39,27 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). + .. math-improvements Improve performance of long operations where one of the operands fits into an int. + diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -609,8 +609,14 @@ warnoptions.extend(pythonwarnings.split(',')) if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -977,6 +977,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -109,6 +109,7 @@ '_promote' : 'interp_magic._promote', 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -211,3 +211,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -644,6 +644,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', @@ -1180,6 +1181,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1726,6 +1731,12 @@ w_mod = state.fixup_extension(name, path) return w_mod +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -47,6 +47,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -261,6 +261,7 @@ if w_obj is not None: if w_obj is not w_marker_deallocating: return w_obj + type_name = rffi.charp2str(cts.cast('char*', ref.c_ob_type.c_tp_name)) fatalerror( "*** Invalid usage of a dying CPython object ***\n" "\n" @@ -275,7 +276,8 @@ "freed, making that reference point to garbage.\n" ">>> PyPy could contain some workaround to still work if\n" "you are lucky, but it is not done so far; better fix the bug in\n" - "the CPython extension.") + "the CPython extension.\n" + ">>> This object is of type '%s'" % (type_name,)) # This reference is not yet a real interpreter object. # Realize it. diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyInt_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -421,6 +421,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -366,10 +366,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -398,28 +402,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -211,9 +211,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -576,6 +576,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in e.value.message + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in excinfo.value.message - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in e.value.message +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in excinfo.value.message - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) From pypy.commits at gmail.com Thu Dec 13 14:19:52 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 13 Dec 2018 11:19:52 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Make ctypes tests more py3-friendly Message-ID: <5c12b0d8.1c69fb81.e18f7.a4b8@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95480:da6688b781d8 Date: 2018-12-12 22:52 +0000 http://bitbucket.org/pypy/pypy/changeset/da6688b781d8/ Log: Make ctypes tests more py3-friendly diff --git a/extra_tests/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py --- a/extra_tests/ctypes_tests/test_array.py +++ b/extra_tests/ctypes_tests/test_array.py @@ -2,7 +2,7 @@ from ctypes import * def test_slice(): - values = range(5) + values = list(range(5)) numarray = c_int * 5 na = numarray(*(c_int(x) for x in values)) @@ -14,7 +14,7 @@ def test_init_again(): sz = (c_char * 3)() addr1 = addressof(sz) - sz.__init__(*"foo") + sz.__init__(*b"foo") addr2 = addressof(sz) assert addr1 == addr2 @@ -33,18 +33,18 @@ A = c_char * 10 TP = POINTER(A) x = TP(A()) - assert x[0] != '' + assert x[0] != b'' A = c_wchar * 10 TP = POINTER(A) x = TP(A()) - assert x[0] != '' + assert x[0] != b'' def test_output_simple_array(): A = c_char * 10 AA = A * 10 aa = AA() - assert aa[0] != '' + assert aa[0] != b'' def test_output_complex_test(): class Car(Structure): @@ -52,13 +52,13 @@ ("speed", c_float), ("owner", c_char * 10)] - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" + assert isinstance(Car(b"abcdefghi", 42.0, b"12345").brand, bytes) + assert Car(b"abcdefghi", 42.0, b"12345").brand == b"abcdefghi" + assert Car(b"abcdefghio", 42.0, b"12345").brand == b"abcdefghio" with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") + Car(b"abcdefghiop", 42.0, b"12345") A = Car._fields_[2][1] TP = POINTER(A) x = TP(A()) - assert x[0] != '' + assert x[0] != b'' diff --git a/extra_tests/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py --- a/extra_tests/ctypes_tests/test_buffers.py +++ b/extra_tests/ctypes_tests/test_buffers.py @@ -11,26 +11,26 @@ assert sizeof(b) == 33 * sizeof(c_char) assert type(b[0]) is str - b = create_string_buffer("abc") + b = create_string_buffer(b"abc") assert len(b) == 4 # trailing nul char assert sizeof(b) == 4 * sizeof(c_char) assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" + assert b[0] == b"a" + assert b[:] == b"abc\0" def test_from_buffer(): - b1 = bytearray("abcde") + b1 = bytearray(b"abcde") b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" + assert b[2] == b"c" # - b1 = bytearray("abcd") + b1 = bytearray(b"abcd") b = c_int.from_buffer(b1) assert b.value in (1684234849, # little endian 1633837924) # big endian def test_from_buffer_keepalive(): # Issue #2878 - b1 = bytearray("ab") + b1 = bytearray(b"ab") array = (c_uint16 * 32)() array[6] = c_uint16.from_buffer(b1) # this is also what we get on CPython. I don't think it makes diff --git a/extra_tests/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py --- a/extra_tests/ctypes_tests/test_callbacks.py +++ b/extra_tests/ctypes_tests/test_callbacks.py @@ -44,8 +44,8 @@ (c_float, -math.e), (c_double, 3.14), (c_double, -3.14), - (c_char, "x"), - (c_char, "a"), + (c_char, b"x"), + (c_char, b"a"), ]) @pytest.mark.parametrize('functype', functypes) def test_types(typ, arg, functype): diff --git a/extra_tests/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py --- a/extra_tests/ctypes_tests/test_extra.py +++ b/extra_tests/ctypes_tests/test_extra.py @@ -54,20 +54,20 @@ def test_char_p(): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" + x = c_char_p(b"hello\x00world") + assert x.value == b"hello" + x.value = b"world" + assert x.value == b"world" p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + assert p[0] == x.value == b"world" + p[0] = b"other" + assert x.value == p.contents.value == p[0] == b"other" myarray = (c_char_p * 10)() - myarray[7] = "hello" + myarray[7] = b"hello" assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + assert myarray[7] == b"hello" def test_struct(): class tagpoint(Structure): @@ -113,33 +113,33 @@ def test_char_array(): a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' + a[0] = b'x' + a[1] = b'y' + assert a.value == b'xy' + a[2] = b'z' + assert a.value == b'xyz' b = create_string_buffer(3) assert type(b) is type(a) assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = b"nxw" + assert b[0] == b'n' + assert b[1] == b'x' + assert b[2] == b'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = b"?" + assert b[0] == b'?' + assert b[1] == b'\x00' + assert b[2] == b'w' class S(Structure): _fields_ = [('p', POINTER(c_char))] s = S() s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s.p.contents.value = b'!' + assert b.value == b'!' assert len(create_string_buffer(0)) == 0 @@ -168,7 +168,7 @@ assert not c_int(0) # a bit strange, if you ask me assert c_int(-1) assert not c_byte(0) - assert not c_char('\x00') # hum + assert not c_char(b'\x00') # hum assert not c_float(0.0) assert not c_double(0.0) assert not c_ulonglong(0) @@ -192,18 +192,18 @@ # automatic conversions to c_char_p func.argtypes = [c_char_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func(create_string_buffer("hello")) == "hello" + assert func(b"hello") == b"hello" + assert func(c_char_p(b"hello")) == b"hello" + assert func((c_char * 6)(*b"hello")) == b"hello" + assert func(create_string_buffer(b"hello")) == b"hello" # automatic conversions to c_void_p func.argtypes = [c_void_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func((c_byte * 6)(104,101,108,108,111)) =="hello" - assert func(create_string_buffer("hello")) == "hello" + assert func(b"hello") == b"hello" + assert func(c_char_p(b"hello")) == b"hello" + assert func((c_char * 6)(*b"hello")) == b"hello" + assert func((c_byte * 6)(104,101,108,108,111)) ==b"hello" + assert func(create_string_buffer(b"hello")) == b"hello" def test_varsize_cast(): import struct diff --git a/extra_tests/ctypes_tests/test_functions.py b/extra_tests/ctypes_tests/test_functions.py --- a/extra_tests/ctypes_tests/test_functions.py +++ b/extra_tests/ctypes_tests/test_functions.py @@ -12,7 +12,7 @@ f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_char result = f(0, 0, 0, 0, 0, 0) - assert result == '\x00' + assert result == b'\x00' def test_boolresult(dll): f = dll._testfunc_i_bhilfd @@ -43,10 +43,10 @@ f.restype = c_char f.argtypes = [POINTER(c_char_p)] # - s = c_char_p('hello world') + s = c_char_p(b'hello world') ps = pointer(s) - assert f(ps) == 'h' - assert f(s) == 'h' # automatic conversion from char** to char* + assert f(ps) == b'h' + assert f(s) == b'h' # automatic conversion from char** to char* ################################################################ @@ -54,8 +54,8 @@ f = dll.my_strchr f.argtypes = [c_char_p] f.restype = c_char_p - result = f("abcd", ord("b")) - assert result == "bcd" + result = f(b"abcd", ord("b")) + assert result == b"bcd" @pytest.mark.pypy_only def test_keepalive_buffers(monkeypatch, dll): @@ -72,8 +72,8 @@ return orig__call_funcptr(funcptr, *newargs) monkeypatch.setattr(f, '_call_funcptr', _call_funcptr) # - result = f("abcd", ord("b")) - assert result == "bcd" + result = f(b"abcd", ord("b")) + assert result == b"bcd" def test_caching_bug_1(dll): # the same test as test_call_some_args, with two extra lines @@ -82,17 +82,17 @@ f = dll.my_strchr f.argtypes = [c_char_p, c_int] f.restype = c_char_p - result = f("abcd", ord("b")) - assert result == "bcd" - result = f("abcd", ord("b"), 42) - assert result == "bcd" + result = f(b"abcd", ord("b")) + assert result == b"bcd" + result = f(b"abcd", ord("b"), 42) + assert result == b"bcd" def test_argument_conversion_and_checks(dll): #This test is designed to check for segfaults if the wrong type of argument is passed as parameter strlen = dll.my_strchr strlen.argtypes = [c_char_p, c_int] strlen.restype = c_char_p - assert strlen("eggs", ord("g")) == "ggs" + assert strlen(b"eggs", ord("g")) == b"ggs" # Should raise ArgumentError, not segfault with pytest.raises(ArgumentError): @@ -205,10 +205,10 @@ get_data_signature = ('test_issue1655', dll) get_data = get_data_prototype(get_data_signature, get_data_paramflag) - assert get_data('testing!') == 4 + assert get_data(b'testing!') == 4 get_data.errcheck = ret_list_p(1) - assert get_data('testing!') == [-1, -2, -3, -4] + assert get_data(b'testing!') == [-1, -2, -3, -4] def test_issue2533(tmpdir): import cffi diff --git a/extra_tests/ctypes_tests/test_guess_argtypes.py b/extra_tests/ctypes_tests/test_guess_argtypes.py --- a/extra_tests/ctypes_tests/test_guess_argtypes.py +++ b/extra_tests/ctypes_tests/test_guess_argtypes.py @@ -18,7 +18,7 @@ assert guess(13) == c_int assert guess(0) == c_int - assert guess('xca') == c_char_p + assert guess(b'xca') == c_char_p assert guess(None) == c_void_p assert guess(c_int(3)) == c_int assert guess(u'xca') == c_wchar_p diff --git a/extra_tests/ctypes_tests/test_keepalive.py b/extra_tests/ctypes_tests/test_keepalive.py --- a/extra_tests/ctypes_tests/test_keepalive.py +++ b/extra_tests/ctypes_tests/test_keepalive.py @@ -101,12 +101,12 @@ p = pointer(x) assert p._objects == {'1':x} p[0] = y - assert p._objects.keys() == ['1'] + assert list(p._objects.keys()) == ['1'] assert p._objects['1'].value == 3 @pytest.mark.pypy_only def test_primitive(): - assert c_char_p("abc")._objects._buffer[0] == "a" + assert c_char_p(b"abc")._objects._buffer[0] == b"a" assert c_int(3)._objects is None def test_pointer_to_pointer(): @@ -222,13 +222,12 @@ def test_c_char_p(): n = 2 - xs = "hello" * n + xs = b"hello" * n x = c_char_p(xs) del xs import gc; gc.collect() - print 'x =', repr(x) - assert x.value == 'hellohello' - assert x._objects == 'hellohello' + assert x.value == b'hellohello' + assert x._objects == b'hellohello' # class datum(Structure): _fields_ = [ @@ -242,7 +241,7 @@ ] for wrap in [False, True]: n = 2 - xs = "hello" * n + xs = b"hello" * n if wrap: xs = c_char_p(xs) dat = datum() @@ -250,19 +249,15 @@ dat.dsize = 15 del xs import gc; gc.collect() - print 'dat.dptr =', repr(dat.dptr) - print 'dat._objects =', repr(dat._objects) - assert dat.dptr == "hellohello" - assert dat._objects.keys() == ['0'] + assert dat.dptr == b"hellohello" + assert list(dat._objects.keys()) == ['0'] - xs = "hello" * n + xs = b"hello" * n if wrap: xs = c_char_p(xs) dat = union() dat.dptr = xs del xs import gc; gc.collect() - print 'dat.dptr =', repr(dat.dptr) - print 'dat._objects =', repr(dat._objects) - assert dat.dptr == "hellohello" - assert dat._objects.keys() == ['0'] + assert dat.dptr == b"hellohello" + assert list(dat._objects.keys()) == ['0'] diff --git a/extra_tests/ctypes_tests/test_prototypes.py b/extra_tests/ctypes_tests/test_prototypes.py --- a/extra_tests/ctypes_tests/test_prototypes.py +++ b/extra_tests/ctypes_tests/test_prototypes.py @@ -29,31 +29,31 @@ def test_kwargs(dll): proto = CFUNCTYPE(c_char_p, c_char_p, c_int) - paramflags = (1, 'text', "tavino"), (1, 'letter', ord('v')) + paramflags = (1, 'text', b"tavino"), (1, 'letter', ord('v')) func = proto(('my_strchr', dll), paramflags) assert func.argtypes == (c_char_p, c_int) assert func.restype == c_char_p - result = func("abcd", ord('b')) - assert result == "bcd" + result = func(b"abcd", ord('b')) + assert result == b"bcd" result = func() - assert result == "vino" + assert result == b"vino" - result = func("grapevine") - assert result == "vine" + result = func(b"grapevine") + assert result == b"vine" - result = func(text="grapevine") - assert result == "vine" + result = func(text=b"grapevine") + assert result == b"vine" result = func(letter=ord('i')) - assert result == "ino" + assert result == b"ino" - result = func(letter=ord('p'), text="impossible") - assert result == "possible" + result = func(letter=ord('p'), text=b"impossible") + assert result == b"possible" - result = func(text="impossible", letter=ord('p')) - assert result == "possible" + result = func(text=b"impossible", letter=ord('p')) + assert result == b"possible" def test_array_to_ptr_wrongtype(dll): ARRAY = c_byte * 8 diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -22,8 +22,8 @@ ("age", c_int)) # short enough - p = Person("123456", 6) - assert p.name == "123456" + p = Person(b"123456", 6) + assert p.name == b"123456" assert p.age == 6 def test___init__(): @@ -32,11 +32,11 @@ ("age", c_int)) def __init__(self, name, surname, age): - self.name = name + ' ' + surname + self.name = name + b' ' + surname self.age = age - p = Person("John", "Doe", 25) - assert p.name == "John Doe" + p = Person(b"John", b"Doe", 25) + assert p.name == b"John Doe" assert p.age == 25 def test_setattr(): diff --git a/extra_tests/ctypes_tests/test_unions.py b/extra_tests/ctypes_tests/test_unions.py --- a/extra_tests/ctypes_tests/test_unions.py +++ b/extra_tests/ctypes_tests/test_unions.py @@ -8,9 +8,9 @@ stuff = Stuff() stuff.y = ord('x') | (ord('z') << 24) if sys.byteorder == 'little': - assert stuff.x == 'x' + assert stuff.x == b'x' else: - assert stuff.x == 'z' + assert stuff.x == b'z' def test_union_of_structures(): class Stuff(Structure): diff --git a/extra_tests/ctypes_tests/test_values.py b/extra_tests/ctypes_tests/test_values.py --- a/extra_tests/ctypes_tests/test_values.py +++ b/extra_tests/ctypes_tests/test_values.py @@ -5,6 +5,6 @@ A testcase which accesses *values* in a dll. """ a_string = (c_char * 16).in_dll(dll, "a_string") - assert a_string.raw == "0123456789abcdef" - a_string[15] = '$' + assert a_string.raw == b"0123456789abcdef" + a_string[15:16] = b'$' assert dll.get_a_string_char(15) == ord('$') From pypy.commits at gmail.com Thu Dec 13 16:38:21 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 13 Dec 2018 13:38:21 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Close branch cleanup-test_lib_pypy Message-ID: <5c12d14d.1c69fb81.4ad0a.3400@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95485:11677e1769a0 Date: 2018-12-13 21:37 +0000 http://bitbucket.org/pypy/pypy/changeset/11677e1769a0/ Log: Close branch cleanup-test_lib_pypy From pypy.commits at gmail.com Thu Dec 13 16:38:35 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 13 Dec 2018 13:38:35 -0800 (PST) Subject: [pypy-commit] pypy default: Merged in cleanup-test_lib_pypy (pull request #633) Message-ID: <5c12d15b.1c69fb81.d4568.e59a@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95486:50d481bd328f Date: 2018-12-13 21:37 +0000 http://bitbucket.org/pypy/pypy/changeset/50d481bd328f/ Log: Merged in cleanup-test_lib_pypy (pull request #633) Cleanup test_lib_pypy diff too long, truncating to 2000 out of 9161 lines diff --git a/pypy/module/test_lib_pypy/ctypes_tests/__init__.py b/extra_tests/ctypes_tests/__init__.py rename from pypy/module/test_lib_pypy/ctypes_tests/__init__.py rename to extra_tests/ctypes_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/extra_tests/ctypes_tests/_ctypes_test.c rename from pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c rename to extra_tests/ctypes_tests/_ctypes_test.c diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py rename from pypy/module/test_lib_pypy/ctypes_tests/conftest.py rename to extra_tests/ctypes_tests/conftest.py --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -3,10 +3,6 @@ import sys import os -def pytest_ignore_collect(path): - if '__pypy__' not in sys.builtin_module_names: - return True - # XXX: copied from pypy/tool/cpyext/extbuild.py if os.name != 'nt': so_ext = 'so' @@ -85,8 +81,7 @@ return outputfilename # end copy -def compile_so_file(): - udir = pytest.ensuretemp('_ctypes_test') +def compile_so_file(udir): cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': @@ -96,8 +91,12 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) -# we need to run after the "tmpdir" plugin which installs pytest.ensuretemp - at pytest.mark.trylast -def pytest_configure(config): - global sofile - sofile = compile_so_file() + at pytest.fixture(scope='session') +def sofile(tmpdir_factory): + udir = tmpdir_factory.mktemp('_ctypes_test') + return str(compile_so_file(udir)) + + at pytest.fixture +def dll(sofile): + from ctypes import CDLL + return CDLL(str(sofile)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/extra_tests/ctypes_tests/support.py rename from pypy/module/test_lib_pypy/ctypes_tests/support.py rename to extra_tests/ctypes_tests/support.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_anon.py rename to extra_tests/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/extra_tests/ctypes_tests/test_anon.py @@ -1,86 +1,55 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestAnon(BaseCTypesTestChecker): + at pytest.mark.pypy_only +def test_nested(): + class ANON_S(Structure): + _fields_ = [("a", c_int)] - def test_anon(self): - class ANON(Union): - _fields_ = [("a", c_int), - ("b", c_int)] + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON), - ("y", c_int)] - _anonymous_ = ["_"] + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) + assert Y.x.offset == 0 + assert Y.a.offset == sizeof(c_int) + assert Y.b.offset == sizeof(c_int) + assert Y._.offset == sizeof(c_int) + assert Y.y.offset == sizeof(c_int) * 2 - assert ANON.a.offset == 0 - assert ANON.b.offset == 0 + assert Y._names_ == ['x', 'a', 'b', 'y'] - def test_anon_nonseq(self): - # TypeError: _anonymous_ must be a sequence - with pytest.raises(TypeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": 42}) +def test_anonymous_fields_on_instance(): + # this is about the *instance-level* access of anonymous fields, + # which you'd guess is the most common, but used not to work + # (issue #2230) - def test_anon_nonmember(self): - # AttributeError: type object 'Name' has no attribute 'x' - with pytest.raises(AttributeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": ["x"]}) + class B(Structure): + _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] + class A(Structure): + _anonymous_ = ["b"] + _fields_ = [("b", B)] - def test_nested(self): - class ANON_S(Structure): - _fields_ = [("a", c_int)] + a = A() + a.x = 5 + assert a.x == 5 + assert a.b.x == 5 + a.b.x += 1 + assert a.x == 6 - class ANON_U(Union): - _fields_ = [("_", ANON_S), - ("b", c_int)] - _anonymous_ = ["_"] + class C(Structure): + _anonymous_ = ["a"] + _fields_ = [("v", c_int), ("a", A)] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON_U), - ("y", c_int)] - _anonymous_ = ["_"] - - assert Y.x.offset == 0 - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) - assert Y._.offset == sizeof(c_int) - assert Y.y.offset == sizeof(c_int) * 2 - - assert Y._names_ == ['x', 'a', 'b', 'y'] - - def test_anonymous_fields_on_instance(self): - # this is about the *instance-level* access of anonymous fields, - # which you'd guess is the most common, but used not to work - # (issue #2230) - - class B(Structure): - _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] - class A(Structure): - _anonymous_ = ["b"] - _fields_ = [("b", B)] - - a = A() - a.x = 5 - assert a.x == 5 - assert a.b.x == 5 - a.b.x += 1 - assert a.x == 6 - - class C(Structure): - _anonymous_ = ["a"] - _fields_ = [("v", c_int), ("a", A)] - - c = C() - c.v = 3 - c.y = -8 - assert c.v == 3 - assert c.y == c.a.y == c.a.b.y == -8 - assert not hasattr(c, 'b') + c = C() + c.v = 3 + c.y = -8 + assert c.v == 3 + assert c.y == c.a.y == c.a.b.y == -8 + assert not hasattr(c, 'b') diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_array.py rename to extra_tests/ctypes_tests/test_array.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py +++ b/extra_tests/ctypes_tests/test_array.py @@ -1,177 +1,64 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -formats = "bBhHiIlLqQfd" +def test_slice(): + values = list(range(5)) + numarray = c_int * 5 -formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ - c_long, c_ulonglong, c_float, c_double + na = numarray(*(c_int(x) for x in values)) -class TestArray(BaseCTypesTestChecker): - def test_simple(self): - # create classes holding simple numeric types, and check - # various properties. + assert list(na[0:0]) == [] + assert list(na[:]) == values + assert list(na[:10]) == values - init = range(15, 25) +def test_init_again(): + sz = (c_char * 3)() + addr1 = addressof(sz) + sz.__init__(*b"foo") + addr2 = addressof(sz) + assert addr1 == addr2 - for fmt in formats: - alen = len(init) - int_array = ARRAY(fmt, alen) +def test_array_of_structures(): + class X(Structure): + _fields_ = [('x', c_int), ('y', c_int)] - ia = int_array(*init) - # length of instance ok? - assert len(ia) == alen + Y = X * 2 + y = Y() + x = X() + x.y = 3 + y[1] = x + assert y[1].y == 3 - # slot values ok? - values = [ia[i] for i in range(len(init))] - assert values == init +def test_output_simple(): + A = c_char * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # change the items - from operator import setitem - new_values = range(42, 42+alen) - [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(len(init))] - assert values == new_values + A = c_wchar * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # are the items initialized to 0? - ia = int_array() - values = [ia[i] for i in range(len(init))] - assert values == [0] * len(init) +def test_output_simple_array(): + A = c_char * 10 + AA = A * 10 + aa = AA() + assert aa[0] != b'' - # Too many in itializers should be caught - with pytest.raises(IndexError): - int_array(*range(alen*2)) +def test_output_complex_test(): + class Car(Structure): + _fields_ = [("brand", c_char * 10), + ("speed", c_float), + ("owner", c_char * 10)] - CharArray = ARRAY(c_char, 3) + assert isinstance(Car(b"abcdefghi", 42.0, b"12345").brand, bytes) + assert Car(b"abcdefghi", 42.0, b"12345").brand == b"abcdefghi" + assert Car(b"abcdefghio", 42.0, b"12345").brand == b"abcdefghio" + with pytest.raises(ValueError): + Car(b"abcdefghiop", 42.0, b"12345") - ca = CharArray("a", "b", "c") - - # Should this work? It doesn't: - # CharArray("abc") - with pytest.raises(TypeError): - CharArray("abc") - - assert ca[0] == "a" - assert ca[1] == "b" - assert ca[2] == "c" - assert ca[-3] == "a" - assert ca[-2] == "b" - assert ca[-1] == "c" - - assert len(ca) == 3 - - # slicing is now supported, but not extended slicing (3-argument)! - from operator import getslice, delitem - with pytest.raises(TypeError): - getslice(ca, 0, 1, -1) - - # cannot delete items - with pytest.raises(TypeError): - delitem(ca, 0) - - def test_numeric_arrays(self): - - alen = 5 - - numarray = ARRAY(c_int, alen) - - na = numarray() - values = [na[i] for i in range(alen)] - assert values == [0] * alen - - na = numarray(*[c_int()] * alen) - values = [na[i] for i in range(alen)] - assert values == [0]*alen - - na = numarray(1, 2, 3, 4, 5) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - na = numarray(*map(c_int, (1, 2, 3, 4, 5))) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - def test_slice(self): - values = range(5) - numarray = c_int * 5 - - na = numarray(*(c_int(x) for x in values)) - - assert list(na[0:0]) == [] - assert list(na[:]) == values - assert list(na[:10]) == values - - def test_classcache(self): - assert not ARRAY(c_int, 3) is ARRAY(c_int, 4) - assert ARRAY(c_int, 3) is ARRAY(c_int, 3) - - def test_from_address(self): - # Failed with 0.9.8, reported by JUrner - p = create_string_buffer("foo") - sz = (c_char * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - - def test_init_again(self): - sz = (c_char * 3)() - addr1 = addressof(sz) - sz.__init__(*"foo") - addr2 = addressof(sz) - assert addr1 == addr2 - - try: - create_unicode_buffer - except NameError: - pass - else: - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - -class TestSophisticatedThings(BaseCTypesTestChecker): - def test_array_of_structures(self): - class X(Structure): - _fields_ = [('x', c_int), ('y', c_int)] - - Y = X * 2 - y = Y() - x = X() - x.y = 3 - y[1] = x - assert y[1].y == 3 - - def test_output_simple(self): - A = c_char * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - A = c_wchar * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - def test_output_simple_array(self): - A = c_char * 10 - AA = A * 10 - aa = AA() - assert aa[0] != '' - - def test_output_complex_test(self): - class Car(Structure): - _fields_ = [("brand", c_char * 10), - ("speed", c_float), - ("owner", c_char * 10)] - - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" - with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") - - A = Car._fields_[2][1] - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = Car._fields_[2][1] + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_base.py rename to extra_tests/ctypes_tests/test_base.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py +++ b/extra_tests/ctypes_tests/test_base.py @@ -1,26 +1,24 @@ -from .support import WhiteBoxTests - +import pytest from ctypes import * -# WhiteBoxTests +pytestmark = pytest.mark.pypy_only -class TestCTypesBase(WhiteBoxTests): - def test_pointer(self): - p = pointer(pointer(c_int(2))) - x = p[0] - assert x._base is p +def test_pointer(): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p - def test_structure(self): - class X(Structure): - _fields_ = [('x', POINTER(c_int)), - ('y', POINTER(c_int))] +def test_structure(): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] - x = X() - assert x.y._base is x - assert x.y._index == 1 + x = X() + assert x.y._base is x + assert x.y._index == 1 - def test_array(self): - X = POINTER(c_int) * 24 - x = X() - assert x[16]._base is x - assert x[16]._index == 16 +def test_array(): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/extra_tests/ctypes_tests/test_bitfields.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py rename to extra_tests/ctypes_tests/test_bitfields.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py +++ b/extra_tests/ctypes_tests/test_bitfields.py @@ -1,249 +1,19 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -import os -import ctypes -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types +def test_set_fields_attr(): + class A(Structure): + pass + A._fields_ = [("a", c_byte), ("b", c_ubyte)] +def test_set_fields_attr_bitfields(): + class A(Structure): + pass + A._fields_ = [("a", POINTER(A)), ("b", c_ubyte, 4)] -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - func = CDLL(_ctypes_test).unpack_bitfields - func.argtypes = POINTER(BITS), c_char - mod.func = func - - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - - -class TestC: - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - def test_shorts(self): - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - -class TestBitField: - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - x.a, x.b, x.c = -1, 7, -1 - assert (x.a, x.b, x.c) == (-1, 7, -1) - - x = X() - x.a, x.b, x.c = -1, -7, -1 - assert (x.a, x.b, x.c) == (-1, -7, -1) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - assert (x.a, x.b, x.c) == (0, 0, 0) - x.a, x.b, x.c = 7, 2305843009213693953, 7 - assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ)*2 - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, -1, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, -1, 0) - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 7, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 7, 0) - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char_p') - - result = self.fail_fields(("a", c_void_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_void_p') - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - assert result == (TypeError, 'bit fields not allowed for type LP_c_int') - - result = self.fail_fields(("a", c_char, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char') - - try: - c_wchar - except NameError: - pass - else: - result = self.fail_fields(("a", c_wchar, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_wchar') - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - assert result == (TypeError, 'bit fields not allowed for type Dummy') - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - result = self.fail_fields(("a", c_typ, 0)) - assert result == (ValueError, 'number of bits invalid for bit field') - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - assert sizeof(X) == sizeof(c_typ) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == 0 - assert X.a1.offset == sizeof(c_short) - assert X.b.offset == sizeof(c_short)*2 - assert X.c.offset == sizeof(c_short)*2 - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == sizeof(c_short)*0 - assert X.b.offset == sizeof(c_short)*1 - assert X.c.offset == sizeof(c_short)*2 - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - import traceback - traceback.print_exc() - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name in ("nt", "ce"): - assert sizeof(X) == sizeof(c_int)*2 - else: - assert sizeof(X) == sizeof(c_int) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - assert sizeof(X) == sizeof(c_int)*2 - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - assert sizeof(X) == sizeof(c_byte) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - def test_set_fields_attr(self): - class A(Structure): - pass - A._fields_ = [("a", c_byte), - ("b", c_ubyte)] - - def test_set_fields_attr_bitfields(self): - class A(Structure): - pass - A._fields_ = [("a", POINTER(A)), - ("b", c_ubyte, 4)] - - def test_set_fields_cycle_fails(self): - class A(Structure): - pass - with pytest.raises(AttributeError): - A._fields_ = [("a", A)] +def test_set_fields_cycle_fails(): + class A(Structure): + pass + with pytest.raises(AttributeError): + A._fields_ = [("a", A)] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py rename to extra_tests/ctypes_tests/test_buffers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py +++ b/extra_tests/ctypes_tests/test_buffers.py @@ -1,76 +1,38 @@ from ctypes import * -from .support import BaseCTypesTestChecker -class TestStringBuffer(BaseCTypesTestChecker): +def test_buffer(): + b = create_string_buffer(32) + assert len(b) == 32 + assert sizeof(b) == 32 * sizeof(c_char) + assert type(b[0]) is str - def test_buffer(self): - b = create_string_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(33L) + assert len(b) == 33 + assert sizeof(b) == 33 * sizeof(c_char) + assert type(b[0]) is str - b = create_string_buffer(33L) - assert len(b) == 33 - assert sizeof(b) == 33 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(b"abc") + assert len(b) == 4 # trailing nul char + assert sizeof(b) == 4 * sizeof(c_char) + assert type(b[0]) is str + assert b[0] == b"a" + assert b[:] == b"abc\0" - b = create_string_buffer("abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" +def test_from_buffer(): + b1 = bytearray(b"abcde") + b = (c_char * 5).from_buffer(b1) + assert b[2] == b"c" + # + b1 = bytearray(b"abcd") + b = c_int.from_buffer(b1) + assert b.value in (1684234849, # little endian + 1633837924) # big endian - def test_string_conversion(self): - b = create_string_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" - - def test_from_buffer(self): - b1 = bytearray("abcde") - b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" - # - b1 = bytearray("abcd") - b = c_int.from_buffer(b1) - assert b.value in (1684234849, # little endian - 1633837924) # big endian - - def test_from_buffer_keepalive(self): - # Issue #2878 - b1 = bytearray("ab") - array = (c_uint16 * 32)() - array[6] = c_uint16.from_buffer(b1) - # this is also what we get on CPython. I don't think it makes - # sense because the array contains just a copy of the number. - assert array._objects == {'6': b1} - - try: - c_wchar - except NameError: - pass - else: - def test_unicode_buffer(self): - b = create_unicode_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_wchar) - assert type(b[0]) is unicode - - b = create_unicode_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - - def test_unicode_conversion(self): - b = create_unicode_buffer("abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - +def test_from_buffer_keepalive(): + # Issue #2878 + b1 = bytearray(b"ab") + array = (c_uint16 * 32)() + array[6] = c_uint16.from_buffer(b1) + # this is also what we get on CPython. I don't think it makes + # sense because the array contains just a copy of the number. + assert array._objects == {'6': b1} diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/extra_tests/ctypes_tests/test_callback_traceback.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py rename to extra_tests/ctypes_tests/test_callback_traceback.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py +++ b/extra_tests/ctypes_tests/test_callback_traceback.py @@ -1,80 +1,35 @@ # derived from test_random_things.py -import py +import pytest + from ctypes import * -import sys -def callback_func(arg): - 42 / arg - raise ValueError(arg) +_rawffi = pytest.importorskip('_rawffi') -class TestCallbackTraceback: - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. +# +# This test makes sure the exception types *and* the exception +# value is printed correctly. + + at pytest.mark.skipif("sys.flags.inspect") +def test_SystemExit(monkeypatch, capsys): + """ + When an exception is raised in a ctypes callback function, the C + code prints a traceback. When SystemExit is raised, the interpreter + normally exits immediately. + """ + def callback_func(arg): + raise SystemExit(42) + def custom_exit(value): + raise Exception("<<>>" % (value,)) + monkeypatch.setattr(_rawffi, 'exit', custom_exit) + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) + out, err = capsys.readouterr() + assert not err + cb2(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead a additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import StringIO - old_stderr = sys.stderr - logger = sys.stderr = StringIO.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() - - def test_ValueError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - assert out.splitlines()[-1] == ( - "ValueError: 42") - - def test_IntegerDivisionError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_FloatDivisionError(self): - cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_TypeErrorDivisionError(self): - cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, "spam") - assert out.splitlines()[-1].startswith( - "TypeError: " - "unsupported operand type(s) for") - - def test_SystemExit(self): - import _rawffi - if sys.flags.inspect: - skip("requires sys.flags.inspect == 0") - def callback_func(arg): - raise SystemExit(42) - def custom_exit(value): - raise Exception("<<>>" % (value,)) - original_exit = _rawffi.exit - try: - _rawffi.exit = custom_exit - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) - out = self.capture_stderr(cb2, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - finally: - _rawffi.exit = original_exit + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py rename to extra_tests/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/extra_tests/ctypes_tests/test_callbacks.py @@ -1,283 +1,194 @@ +import pytest + +import math from ctypes import * -import pytest from .support import BaseCTypesTestChecker -class TestCallbacks(BaseCTypesTestChecker): - functype = CFUNCTYPE - -## def tearDown(self): -## import gc -## gc.collect() - - def callback(self, *args): - self.got_args = args - return args[-1] - - def check_type(self, typ, arg): - unwrapped_types = { - c_float: (float,), - c_double: (float,), - c_char: (str,), - c_char_p: (str,), - c_uint: (int, long), - c_ulong: (int, long), - } - - PROTO = self.functype.im_func(typ, typ) - cfunc = PROTO(self.callback) - result = cfunc(arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (arg,) - assert result == arg - - result2 = cfunc(typ(arg)) - assert type(result2) in unwrapped_types.get(typ, (int, long)) - - PROTO = self.functype.im_func(typ, c_byte, typ) - result = PROTO(self.callback)(-3, arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (-3, arg) - assert result == arg - - ################ - - def test_byte(self): - self.check_type(c_byte, 42) - self.check_type(c_byte, -42) - - def test_ubyte(self): - self.check_type(c_ubyte, 42) - - def test_short(self): - self.check_type(c_short, 42) - self.check_type(c_short, -42) - - def test_ushort(self): - self.check_type(c_ushort, 42) - - def test_int(self): - self.check_type(c_int, 42) - self.check_type(c_int, -42) - - def test_uint(self): - self.check_type(c_uint, 42) - - def test_long(self): - self.check_type(c_long, 42) - self.check_type(c_long, -42) - - def test_ulong(self): - self.check_type(c_ulong, 42) - - def test_longlong(self): - self.check_type(c_longlong, 42) - self.check_type(c_longlong, -42) - - def test_ulonglong(self): - self.check_type(c_ulonglong, 42) - - def test_float(self): - # only almost equal: double -> float -> double - import math - self.check_type(c_float, math.e) - self.check_type(c_float, -math.e) - - def test_double(self): - self.check_type(c_double, 3.14) - self.check_type(c_double, -3.14) - - def test_char(self): - self.check_type(c_char, "x") - self.check_type(c_char, "a") - - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. -## def test_char_p(self): -## self.check_type(c_char_p, "abc") -## self.check_type(c_char_p, "def") - - - @pytest.mark.xfail( - reason="we are less strict about callback return type sanity") - def test_unsupported_restype_1(self): - # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. - # POINTER(c_double), for example, is not supported. - - prototype = self.functype.im_func(POINTER(c_double)) - # The type is checked when the prototype is called - with pytest.raises(TypeError): - prototype(lambda: None) - +functypes = [CFUNCTYPE] try: - WINFUNCTYPE + functypes.append(WINFUNCTYPE) except NameError: pass -else: - class TestStdcallCallbacks(TestCallbacks): - functype = WINFUNCTYPE -################################################################ -class TestSampleCallbacks(BaseCTypesTestChecker): +def callback(*args): + callback.got_args = args + return args[-1] - def test_integrate(self): - # Derived from some then non-working code, posted by David Foster - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) +unwrapped_types = { + c_float: (float,), + c_double: (float,), + c_char: (str,), + c_char_p: (str,), + c_uint: (int, long), + c_ulong: (int, long), + } - # The function prototype called by 'integrate': double func(double); - CALLBACK = CFUNCTYPE(c_double, c_double) + at pytest.mark.parametrize("typ, arg", [ + (c_byte, 42), + (c_byte, -42), + (c_ubyte, 42), + (c_short, 42), + (c_short, -42), + (c_ushort, 42), + (c_int, 42), + (c_int, -42), + (c_uint, 42), + (c_long, 42), + (c_long, -42), + (c_ulong, 42), + (c_longlong, 42), + (c_longlong, -42), + (c_ulonglong, 42), + (c_float, math.e), # only almost equal: double -> float -> double + (c_float, -math.e), + (c_double, 3.14), + (c_double, -3.14), + (c_char, b"x"), + (c_char, b"a"), +]) + at pytest.mark.parametrize('functype', functypes) +def test_types(typ, arg, functype): + PROTO = functype(typ, typ) + cfunc = PROTO(callback) + result = cfunc(arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (arg,) + assert result == arg - # The integrate function itself, exposed from the _ctypes_test dll - integrate = dll.integrate - integrate.argtypes = (c_double, c_double, CALLBACK, c_long) - integrate.restype = c_double + result2 = cfunc(typ(arg)) + assert type(result2) in unwrapped_types.get(typ, (int, long)) - def func(x): - print 'calculating x**2 of',x - return x**2 + PROTO = functype(typ, c_byte, typ) + result = PROTO(callback)(-3, arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (-3, arg) + assert result == arg - result = integrate(0.0, 1.0, CALLBACK(func), 10) - diff = abs(result - 1./3.) + at pytest.mark.parametrize('functype', functypes) +def test_unsupported_restype_1(functype): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. - assert diff < 0.01, "%s not less than 0.01" % diff + prototype = functype(POINTER(c_double)) + # The type is checked when the prototype is called + with pytest.raises(TypeError): + prototype(lambda: None) -################################################################ -class TestMoreCallbacks(BaseCTypesTestChecker): +def test_callback_with_struct_argument(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - def test_callback_with_struct_argument(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] + proto = CFUNCTYPE(c_int, RECT) - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - point.left *= -1 - return point.left+point.top+point.right+point.bottom + def callback(point): + point.left *= -1 + return point.left + point.top + point.right + point.bottom - cbp = proto(callback) + cbp = proto(callback) + rect = RECT(-1000, 100, 10, 1) + res = cbp(rect) + assert res == 1111 + assert rect.left == -1000 # must not have been changed! - rect = RECT(-1000,100,10,1) +def test_callback_from_c_with_struct_argument(dll): + class RECT(Structure): + _fields_ = [("left", c_long), ("top", c_long), + ("right", c_long), ("bottom", c_long)] - res = cbp(rect) + proto = CFUNCTYPE(c_int, RECT) - assert res == 1111 - assert rect.left == -1000 # must not have been changed! + def callback(point): + return point.left + point.top + point.right + point.bottom - def test_callback_from_c_with_struct_argument(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + cbp = proto(callback) + rect = RECT(1000, 100, 10, 1) - class RECT(Structure): - _fields_ = [("left", c_long), ("top", c_long), - ("right", c_long), ("bottom", c_long)] + call_callback_with_rect = dll.call_callback_with_rect + call_callback_with_rect.restype = c_int + call_callback_with_rect.argtypes = [proto, RECT] + res = call_callback_with_rect(cbp, rect) + assert res == 1111 - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - return point.left+point.top+point.right+point.bottom +def test_callback_unsupported_return_struct(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - cbp = proto(callback) - rect = RECT(1000,100,10,1) + proto = CFUNCTYPE(RECT, c_int) + with pytest.raises(TypeError): + proto(lambda r: 0) - call_callback_with_rect = dll.call_callback_with_rect - call_callback_with_rect.restype = c_int - call_callback_with_rect.argtypes = [proto, RECT] - res = call_callback_with_rect(cbp, rect) - assert res == 1111 - def test_callback_unsupported_return_struct(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] +def test_qsort(dll): + PI = POINTER(c_int) + A = c_int*5 + a = A() + for i in range(5): + a[i] = 5-i - proto = CFUNCTYPE(RECT, c_int) - with pytest.raises(TypeError): - proto(lambda r: 0) + assert a[0] == 5 # sanity + def comp(a, b): + a = a.contents.value + b = b.contents.value + return cmp(a,b) + qs = dll.my_qsort + qs.restype = None + CMP = CFUNCTYPE(c_int, PI, PI) + qs.argtypes = (PI, c_size_t, c_size_t, CMP) - def test_qsort(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) - PI = POINTER(c_int) - A = c_int*5 - a = A() - for i in range(5): - a[i] = 5-i + res = list(a) - assert a[0] == 5 # sanity + assert res == [1,2,3,4,5] - def comp(a, b): - a = a.contents.value - b = b.contents.value - return cmp(a,b) - qs = dll.my_qsort - qs.restype = None - CMP = CFUNCTYPE(c_int, PI, PI) - qs.argtypes = (PI, c_size_t, c_size_t, CMP) +def test_pyobject_as_opaque(dll): + def callback(arg): + return arg() - qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) + CTP = CFUNCTYPE(c_int, py_object) + cfunc = dll._testfunc_callback_opaque + cfunc.argtypes = [CTP, py_object] + cfunc.restype = c_int + res = cfunc(CTP(callback), lambda : 3) + assert res == 3 - res = list(a) +def test_callback_void(capsys, dll): + def callback(): + pass - assert res == [1,2,3,4,5] + CTP = CFUNCTYPE(None) + cfunc = dll._testfunc_callback_void + cfunc.argtypes = [CTP] + cfunc.restype = int + cfunc(CTP(callback)) + out, err = capsys.readouterr() + assert (out, err) == ("", "") - def test_pyobject_as_opaque(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - def callback(arg): - return arg() +def test_callback_pyobject(): + def callback(obj): + return obj - CTP = CFUNCTYPE(c_int, py_object) - cfunc = dll._testfunc_callback_opaque - cfunc.argtypes = [CTP, py_object] - cfunc.restype = c_int - res = cfunc(CTP(callback), lambda : 3) - assert res == 3 + FUNC = CFUNCTYPE(py_object, py_object) + cfunc = FUNC(callback) + param = c_int(42) + assert cfunc(param) is param - def test_callback_void(self, capsys): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - - def callback(): - pass - - CTP = CFUNCTYPE(None) - cfunc = dll._testfunc_callback_void - cfunc.argtypes = [CTP] - cfunc.restype = int - cfunc(CTP(callback)) - out, err = capsys.readouterr() - assert (out, err) == ("", "") - - - def test_callback_pyobject(self): - def callback(obj): - return obj - - FUNC = CFUNCTYPE(py_object, py_object) - cfunc = FUNC(callback) - param = c_int(42) - assert cfunc(param) is param - - def test_raise_argumenterror(self): - def callback(x): - pass - FUNC = CFUNCTYPE(None, c_void_p) - cfunc = FUNC(callback) - param = c_uint(42) - with pytest.raises(ArgumentError): - cfunc(param) +def test_raise_argumenterror(): + def callback(x): + pass + FUNC = CFUNCTYPE(None, c_void_p) + cfunc = FUNC(callback) + param = c_uint(42) + with pytest.raises(ArgumentError): + cfunc(param) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/extra_tests/ctypes_tests/test_cast.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_cast.py rename to extra_tests/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/extra_tests/ctypes_tests/test_cast.py @@ -1,106 +1,30 @@ +import pytest + from ctypes import * -import sys, py -from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - mod.lib = CDLL(str(conftest.sofile)) +def test_cast_functype(dll): + # make sure we can cast function type + my_sqrt = dll.my_sqrt + saved_objects = my_sqrt._objects.copy() + sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) + assert sqrt(4.0) == 2.0 + assert not cast(0, CFUNCTYPE(c_int)) + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) -class TestCast(BaseCTypesTestChecker): +def test_cast_argumenterror(): + param = c_uint(42) + with pytest.raises(ArgumentError): + cast(param, c_void_p) - def test_array2pointer(self): - array = (c_int * 3)(42, 17, 2) - - # casting an array to a pointer works. - ptr = cast(array, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - if 2*sizeof(c_short) == sizeof(c_int): - ptr = cast(array, POINTER(c_short)) - if sys.byteorder == "little": - assert [ptr[i] for i in range(6)] == ( - [42, 0, 17, 0, 2, 0]) - else: - assert [ptr[i] for i in range(6)] == ( - [0, 42, 0, 17, 0, 2]) - - def test_address2pointer(self): - array = (c_int * 3)(42, 17, 2) - - address = addressof(array) - ptr = cast(c_void_p(address), POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - ptr = cast(address, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - def test_p2a_objects(self): - py.test.skip("we make copies of strings") - array = (c_char_p * 5)() - assert array._objects is None - array[0] = "foo bar" - assert array._objects == {'0': "foo bar"} - - p = cast(array, POINTER(c_char_p)) - # array and p share a common _objects attribute - assert p._objects is array._objects - assert array._objects == {'0': "foo bar", id(array): array} - p[0] = "spam spam" - assert p._objects == {'0': "spam spam", id(array): array} - assert array._objects is p._objects - p[1] = "foo bar" - assert p._objects == {'1': 'foo bar', '0': "spam spam", id(array): array} - assert array._objects is p._objects - - def test_other(self): - p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) - assert p[:4] == [1,2, 3, 4] - c_int() - assert p[:4] == [1, 2, 3, 4] - p[2] = 96 - assert p[:4] == [1, 2, 96, 4] - c_int() - assert p[:4] == [1, 2, 96, 4] - - def test_char_p(self): - # This didn't work: bad argument to internal function - s = c_char_p("hiho") - - assert cast(cast(s, c_void_p), c_char_p).value == ( - "hiho") - - try: - c_wchar_p - except NameError: - pass - else: - def test_wchar_p(self): - s = c_wchar_p("hiho") - assert cast(cast(s, c_void_p), c_wchar_p).value == ( - "hiho") - - def test_cast_functype(self): - # make sure we can cast function type - my_sqrt = lib.my_sqrt - saved_objects = my_sqrt._objects.copy() - sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) - assert sqrt(4.0) == 2.0 - assert not cast(0, CFUNCTYPE(c_int)) - # - assert sqrt._objects is my_sqrt._objects # on CPython too - my_sqrt._objects.clear() - my_sqrt._objects.update(saved_objects) - - def test_cast_argumenterror(self): - param = c_uint(42) - py.test.raises(ArgumentError, "cast(param, c_void_p)") - - def test_c_bool(self): - x = c_bool(42) - assert x.value is True - x = c_bool(0.0) - assert x.value is False - x = c_bool("") - assert x.value is False - x = c_bool(['yadda']) - assert x.value is True +def test_c_bool(): + x = c_bool(42) + assert x.value is True + x = c_bool(0.0) + assert x.value is False + x = c_bool("") + assert x.value is False + x = c_bool(['yadda']) + assert x.value is True diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py b/extra_tests/ctypes_tests/test_commethods.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py rename to extra_tests/ctypes_tests/test_commethods.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/extra_tests/ctypes_tests/test_errno.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_errno.py rename to extra_tests/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/extra_tests/ctypes_tests/test_errno.py @@ -1,26 +1,19 @@ -import py +import pytest import ctypes -from _ctypes import function +_rawffi = pytest.importorskip('_rawffi') # PyPy-only -try: - import _rawffi -except ImportError: - py.test.skip("app-level test only for PyPy") +def test_errno_saved_and_restored(): + def check(): + assert _rawffi.get_errno() == 42 + assert ctypes.get_errno() == old + check.free_temp_buffers = lambda *args: None + f = ctypes._CFuncPtr() + old = _rawffi.get_errno() + f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO + ctypes.set_errno(42) + f._call_funcptr(check) + assert _rawffi.get_errno() == old + ctypes.set_errno(0) -class TestErrno: - - def test_errno_saved_and_restored(self): - def check(): - assert _rawffi.get_errno() == 42 - assert ctypes.get_errno() == old - check.free_temp_buffers = lambda *args: None - f = function.CFuncPtr() - old = _rawffi.get_errno() - f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO - ctypes.set_errno(42) - f._call_funcptr(check) - assert _rawffi.get_errno() == old - ctypes.set_errno(0) - - # see also test_functions.test_errno +# see also test_functions.test_errno diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_extra.py rename to extra_tests/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/extra_tests/ctypes_tests/test_extra.py @@ -5,244 +5,239 @@ import py from ctypes import * -from .support import BaseCTypesTestChecker -class TestExtra(BaseCTypesTestChecker): - def test_primitive_pointer(self): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 +def test_primitive_pointer(): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 - ## *** the rest is commented out because it should work but occasionally - ## *** trigger a ctypes bug (SourceForge bug #1467852). *** - ## q = pointer(y) - ## pp[0] = q # pp ---> p ---> y = 14 - ## assert y.value == 14 # (^^^ not q! ) - ## assert p.contents.value == 14 - ## assert pp.contents.contents.value == 14 - ## q.contents = x - ## assert pp.contents.contents.value == 14 +## *** the rest is commented out because it should work but occasionally +## *** trigger a ctypes bug (SourceForge bug #1467852). *** +## q = pointer(y) +## pp[0] = q # pp ---> p ---> y = 14 +## assert y.value == 14 # (^^^ not q! ) +## assert p.contents.value == 14 +## assert pp.contents.contents.value == 14 +## q.contents = x +## assert pp.contents.contents.value == 14 - def test_char_p(self): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" +def test_char_p(): + x = c_char_p(b"hello\x00world") + assert x.value == b"hello" + x.value = b"world" + assert x.value == b"world" - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + p = pointer(x) + assert p[0] == x.value == b"world" + p[0] = b"other" + assert x.value == p.contents.value == p[0] == b"other" - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + myarray = (c_char_p * 10)() + myarray[7] = b"hello" + assert isinstance(myarray[7], str) + assert myarray[7] == b"hello" - def test_struct(self): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] +def test_struct(): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 - def test_ptr_array(self): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) +def test_ptr_array(): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 - def test_void_p(self): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 +def test_void_p(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 - def test_char_array(self): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' +def test_char_array(): + a = (c_char * 3)() + a[0] = b'x' + a[1] = b'y' + assert a.value == b'xy' + a[2] = b'z' + assert a.value == b'xyz' - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = b"nxw" + assert b[0] == b'n' + assert b[1] == b'x' + assert b[2] == b'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = b"?" + assert b[0] == b'?' + assert b[1] == b'\x00' + assert b[2] == b'w' - class S(Structure): - _fields_ = [('p', POINTER(c_char))] + class S(Structure): + _fields_ = [('p', POINTER(c_char))] - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s = S() + s.p = b + s.p.contents.value = b'!' + assert b.value == b'!' - assert len(create_string_buffer(0)) == 0 + assert len(create_string_buffer(0)) == 0 - def test_array(self): - a = (c_int * 10)() +def test_array(): + a = (c_int * 10)() - class S(Structure): - _fields_ = [('p', POINTER(c_int))] + class S(Structure): + _fields_ = [('p', POINTER(c_int))] - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] - def test_truth_value(self): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... +def test_truth_value(): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) - assert not c_double(0.0) - assert not c_ulonglong(0) - assert c_ulonglong(2L**42) + assert c_int(12) + assert not c_int(0) # a bit strange, if you ask me + assert c_int(-1) + assert not c_byte(0) + assert not c_char(b'\x00') # hum + assert not c_float(0.0) + assert not c_double(0.0) + assert not c_ulonglong(0) + assert c_ulonglong(2L**42) - assert c_char_p("hello") - assert c_char_p("") - assert not c_char_p(None) + assert c_char_p("hello") + assert c_char_p("") + assert not c_char_p(None) - assert not c_void_p() + assert not c_void_p() - def test_sizeof(self): - x = create_string_buffer(117) - assert sizeof(x) == 117 # assumes that chars are one byte each - x = (c_int * 42)() - assert sizeof(x) == 42 * sizeof(c_int) +def test_sizeof(): + x = create_string_buffer(117) + assert sizeof(x) == 117 # assumes that chars are one byte each + x = (c_int * 42)() + assert sizeof(x) == 42 * sizeof(c_int) - def test_convert_pointers(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - func = dll._testfunc_p_p - func.restype = c_char_p +def test_convert_pointers(dll): + func = dll._testfunc_p_p + func.restype = c_char_p - # automatic conversions to c_char_p - func.argtypes = [c_char_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func(create_string_buffer("hello")) == "hello" + # automatic conversions to c_char_p + func.argtypes = [c_char_p] + assert func(b"hello") == b"hello" + assert func(c_char_p(b"hello")) == b"hello" + assert func((c_char * 6)(*b"hello")) == b"hello" + assert func(create_string_buffer(b"hello")) == b"hello" - # automatic conversions to c_void_p - func.argtypes = [c_void_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func((c_byte * 6)(104,101,108,108,111)) =="hello" - assert func(create_string_buffer("hello")) == "hello" + # automatic conversions to c_void_p + func.argtypes = [c_void_p] + assert func(b"hello") == b"hello" + assert func(c_char_p(b"hello")) == b"hello" + assert func((c_char * 6)(*b"hello")) == b"hello" + assert func((c_byte * 6)(104,101,108,108,111)) ==b"hello" + assert func(create_string_buffer(b"hello")) == b"hello" - def test_varsize_cast(self): - import struct - N = struct.calcsize("l") - x = c_long() - p = cast(pointer(x), POINTER(c_ubyte*N)) - for i, c in enumerate(struct.pack("l", 12345678)): - p.contents[i] = ord(c) - assert x.value == 12345678 +def test_varsize_cast(): + import struct + N = struct.calcsize("l") + x = c_long() + p = cast(pointer(x), POINTER(c_ubyte*N)) + for i, c in enumerate(struct.pack("l", 12345678)): + p.contents[i] = ord(c) + assert x.value == 12345678 - def test_cfunctype_inspection(self): - T = CFUNCTYPE(c_int, c_ubyte) - # T.argtypes and T.restype don't work, must use a dummy instance - assert list(T().argtypes) == [c_ubyte] - assert T().restype == c_int +def test_cfunctype_inspection(): + T = CFUNCTYPE(c_int, c_ubyte) + # T.argtypes and T.restype don't work, must use a dummy instance From pypy.commits at gmail.com Fri Dec 14 11:52:24 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 14 Dec 2018 08:52:24 -0800 (PST) Subject: [pypy-commit] pypy py3.5: More py3 fixes in extra_tests/ Message-ID: <5c13dfc8.1c69fb81.4a3c2.c944@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95493:61f4233c89ab Date: 2018-12-14 16:51 +0000 http://bitbucket.org/pypy/pypy/changeset/61f4233c89ab/ Log: More py3 fixes in extra_tests/ diff --git a/extra_tests/test_cStringIO.py b/extra_tests/test_cStringIO.py deleted file mode 100644 --- a/extra_tests/test_cStringIO.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Tests for the PyPy cStringIO implementation. -""" -from io import StringIO - -data = b"some bytes" - -def test_reset(): - """ - Test that the reset method of cStringIO objects sets the position - marker to the beginning of the stream. - """ - stream = StringIO() - stream.write(data) - assert stream.read() == '' - stream.reset() - assert stream.read() == data - - stream = StringIO(data) - assert stream.read() == data - assert stream.read() == '' - stream.reset() - assert stream.read() == data diff --git a/extra_tests/test_code.py b/extra_tests/test_code.py --- a/extra_tests/test_code.py +++ b/extra_tests/test_code.py @@ -9,11 +9,8 @@ try: mystdout = StringIO() sys.stdout = mystdout - runner.runcode(compile("print 5,;0/0", "", "exec")) + runner.runcode(compile("print(5);0/0", "", "exec")) finally: sys.stdout = old_stdout - if '__pypy__' in sys.builtin_module_names: - assert mystdout.getvalue() == "5\n" - else: - assert mystdout.getvalue() == "5" + assert mystdout.getvalue() == "5\n" diff --git a/extra_tests/test_dbm.py b/extra_tests/test_dbm.py --- a/extra_tests/test_dbm.py +++ b/extra_tests/test_dbm.py @@ -33,7 +33,7 @@ d[123] with pytest.raises(TypeError): 123 in d - with pytest.raises(TypeError): + with pytest.raises(AttributeError): d.has_key(123) with pytest.raises(TypeError): d.setdefault(123, 'xyz') @@ -43,7 +43,7 @@ d.get(123) assert dict(d) == {} d.setdefault('xyz', '123') - assert dict(d) == {'xyz': '123'} + assert dict(d) == {b'xyz': b'123'} d.close() def test_multiple_sets(tmpdir): @@ -52,15 +52,16 @@ d['xyz'] = '12' d['xyz'] = '3' d['xyz'] = '546' - assert dict(d) == {'xyz': '546'} - assert d['xyz'] == '546' + assert dict(d) == {b'xyz': b'546'} + assert d['xyz'] == b'546' @pytest.mark.skipif("'__pypy__' not in sys.modules") def test_extra(): + import _dbm with pytest.raises(TypeError): - dbm.datum(123) + _dbm.datum(123) with pytest.raises(TypeError): - dbm.datum(False) + _dbm.datum(False) def test_null(): db = dbm.open('test', 'c') @@ -68,20 +69,19 @@ db.close() db = dbm.open('test', 'r') - assert db['1'] == 'a\x00b' + assert db['1'] == b'a\x00b' db.close() def test_key_with_empty_value(tmpdir): # this test fails on CPython too (at least on tannit), and the # case shows up when gdbm is not installed and test_anydbm.py # falls back dbm. - pytest.skip("test may fail on CPython too") path = str(tmpdir.join('test_dbm_extra.test_key_with_empty_value')) d = dbm.open(path, 'c') assert 'key_with_empty_value' not in d d['key_with_empty_value'] = '' assert 'key_with_empty_value' in d - assert d['key_with_empty_value'] == '' + assert d['key_with_empty_value'] == b'' d.close() def test_unicode_filename(tmpdir): diff --git a/extra_tests/test_sqlite3.py b/extra_tests/test_sqlite3.py --- a/extra_tests/test_sqlite3.py +++ b/extra_tests/test_sqlite3.py @@ -213,19 +213,10 @@ def test_returning_blob_must_own_memory(con): import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) + con.create_function("returnblob", 0, lambda: memoryview(b"blob")) cur = con.execute("select returnblob()") val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') + assert isinstance(val, bytes) def test_description_after_fetchall(con): cur = con.cursor() @@ -264,7 +255,7 @@ def test_issue1573(con): cur = con.cursor() cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') + assert cur.description[0][0] == u"méil" def test_adapter_exception(con): def cast(obj): diff --git a/extra_tests/test_string.py b/extra_tests/test_string.py deleted file mode 100644 --- a/extra_tests/test_string.py +++ /dev/null @@ -1,46 +0,0 @@ - -""" -Test module for functions in string.py -""" -import pytest - -def test_maketrans(): - import string - assert string.maketrans('', '') == ( - '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12' - '\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0' - '123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu' - 'vwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d' - '\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e' - '\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' - '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0' - '\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1' - '\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2' - '\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3' - '\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') - assert string.maketrans('a', 'b') == ( - '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12' - '\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0' - '123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`bbcdefghijklmnopqrstu' - 'vwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d' - '\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e' - '\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' - '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0' - '\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1' - '\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2' - '\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3' - '\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') - assert string.maketrans('ab', 'cd') == ( - '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12' - '\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0' - '123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`cdcdefghijklmnopqrstu' - 'vwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d' - '\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e' - '\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' - '\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0' - '\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1' - '\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2' - '\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3' - '\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') - with pytest.raises(ValueError): - string.maketrans('aa', '') From pypy.commits at gmail.com Fri Dec 14 22:00:24 2018 From: pypy.commits at gmail.com (Stian Andreassen) Date: Fri, 14 Dec 2018 19:00:24 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: Revert the inplace invert from rshift as it may break things Message-ID: <5c146e48.1c69fb81.b3b97.4ee3@mx.google.com> Author: Stian Andreassen Branch: math-improvements Changeset: r95497:fc1d97a72983 Date: 2018-12-15 03:56 +0100 http://bitbucket.org/pypy/pypy/changeset/fc1d97a72983/ Log: Revert the inplace invert from rshift as it may break things diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1231,43 +1231,30 @@ raise ValueError("negative shift count") elif int_other == 0: return self - invert = False if self.sign == -1 and not dont_invert: - first = self.digit(0) - if first == 0: - a = self.invert().rshift(int_other) - return a.invert() - invert = True + a = self.invert().rshift(int_other) + return a.invert() wordshift = int_other / SHIFT - loshift = int_other % SHIFT newsize = self.numdigits() - wordshift if newsize <= 0: - if invert: - return ONENEGATIVERBIGINT - else: - return NULLRBIGINT - - + return NULLRBIGINT + + loshift = int_other % SHIFT hishift = SHIFT - loshift z = rbigint([NULLDIGIT] * newsize, self.sign, newsize) i = 0 while i < newsize: - digit = self.udigit(wordshift) - if i == 0 and invert and wordshift == 0: - digit -= 1 - newdigit = (digit >> loshift) + newdigit = (self.digit(wordshift) >> loshift) if i+1 < newsize: - newdigit |= (self.udigit(wordshift+1) << hishift) + newdigit |= (self.digit(wordshift+1) << hishift) z.setdigit(i, newdigit) i += 1 wordshift += 1 - if invert: - z.setdigit(0, z.digit(0)+1) z._normalize() return z - rshift._always_inline_ = 'try' # It's so fast that it's always beneficial. - + rshift._always_inline_ = 'try' # It's so fast that it's always benefitial. + @jit.elidable def rqshift(self, int_other): wordshift = int_other / SHIFT diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -652,6 +652,9 @@ # test special optimization case in rshift: assert rbigint.fromlong(-(1 << 100)).rshift(5).tolong() == -(1 << 100) >> 5 + # Chek value accuracy. + assert rbigint.fromlong(18446744073709551615L).rshift(1).tolong() == 18446744073709551615L >> 1 + def test_qshift(self): for x in range(10): for y in range(1, 161, 16): From pypy.commits at gmail.com Fri Dec 14 22:00:30 2018 From: pypy.commits at gmail.com (Stian Andreassen) Date: Fri, 14 Dec 2018 19:00:30 -0800 (PST) Subject: [pypy-commit] pypy math-improvements: merge heads Message-ID: <5c146e4e.1c69fb81.5ba36.5a3d@mx.google.com> Author: Stian Andreassen Branch: math-improvements Changeset: r95498:a344a7cb2d95 Date: 2018-12-15 03:59 +0100 http://bitbucket.org/pypy/pypy/changeset/a344a7cb2d95/ Log: merge heads diff too long, truncating to 2000 out of 3510 lines diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -182,6 +182,8 @@ module = category[:i] klass = category[i+1:] try: + if not module: + raise ImportError # instead of the ValueError we'd get m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -26,6 +26,12 @@ from pyrepl.fancy_termios import tcgetattr, tcsetattr from pyrepl.console import Console, Event from pyrepl import unix_eventqueue +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -416,6 +422,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError), err: diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -4,7 +4,7 @@ .. contents:: This document gives an overview of the goals and architecture of PyPy. If you're -interested in :ref:`using PyPy ` or :ref:`hacking on it `, +interested in :ref:`using PyPy ` or hacking on it, have a look at our :ref:`getting started ` section. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -192,10 +192,10 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('temp_index', 'PyPy.tex', u'PyPy Documentation', - u'The PyPy Project', 'manual'), -] +#latex_documents = [ +# ('temp_index', 'PyPy.tex', u'PyPy Documentation', +# u'The PyPy Project', 'manual'), +#] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -212,7 +212,7 @@ #latex_appendices = [] # If false, no module index is generated. -latex_use_modindex = False +#latex_use_modindex = False # Example configuration for intersphinx: refer to the Python standard library. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -29,6 +29,7 @@ :maxdepth: 1 introduction + architecture install build windows @@ -59,7 +60,6 @@ :maxdepth: 2 contributing - architecture configuration project-ideas project-documentation diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst --- a/pypy/doc/project-documentation.rst +++ b/pypy/doc/project-documentation.rst @@ -28,7 +28,6 @@ .. toctree:: :hidden: - architecture coding-guide sprint-reports extradoc diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -111,9 +111,12 @@ -------------- Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified. -Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy -test suite. We could use help analyzing the failures and fixing them either -as patches to upstream NumPy, or as fixes to PyPy. +Release PyPy2.7-v6.0 still fails about 10 of the ~6000 test in the NumPy +test suite. We need to improve our ctypes structure -> memoryview conversions_, +and to refactor the way `NumPy adds docstrings`_. + +.. _conversions: https://bitbucket.org/pypy/pypy/issues/2930 +.. _`NumPy adds docstrings`: https://github.com/numpy/numpy/issues/10167 We also are looking for help in how to hijack NumPy dtype conversion and ufunc calls to allow the JIT to make them fast, using our internal _numpypy @@ -165,8 +168,33 @@ Or maybe not. We can also play around with the idea of using a single representation: as a byte string in utf-8. (This idea needs some extra logic -for efficient indexing, like a cache.) +for efficient indexing, like a cache.) Work has begun on the ``unicode-utf`` +and ``unicode-utf8-py3`` branches. More is needed, for instance there are +SIMD optimizations that are not yet used. +Convert RPython to Python3 +-------------------------- + +The world is moving on, we should too. + +Improve performance +------------------- + +* Make uninlined Python-level calls faster +* Switch to a `sea-of-nodes`_ IR, or a `Lua-Jit`_-like IR which iterates on + on the sea-of-nodes approach +* Use real register-allocation +* Improve instruction selection / scheduling +* Create a hybrid tracing/method JIT + +.. _`sea-of-nodes`: https://darksi.de/d.sea-of-nodes/ +.. _`Lua-JIT`: http://wiki.luajit.org/SSA-IR-2.0 + +Improve warmup +-------------- +* Interpreter speed-ups +* Optimize while tracing +* Cache information between runs Translation Toolchain --------------------- @@ -234,6 +262,27 @@ .. _runner: http://speed.pypy.org .. _`CPython site`: https://speed.python.org/ + +Interfacing with C +------------------ + +While we could make ``cpyext`` faster_, we would also like to explore other +ideas. It seems cffi is only appropriate for small to medium-sized extensions, +and it is hard to imagine NumPy abandoning the C-API. Here are a few ideas: +* Extend Cython to have a backend that can be understood by the JIT +* Collaborate with C-extension authors to ensure full PyPy support (see below) +* Put PyPy compatible packages on PyPI and in conda + + +.. _faster: https://morepypy.blogspot.com/2018/09#next-steps + +Support more platforms +---------------------- + +We have a plan for a `Windows 64`_ port. + +.. _`Windows 64`: windows.html#what-is-missing-for-a-full-64-bit-translation + ====================================== Make more python modules pypy-friendly ====================================== diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,8 +39,27 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). + .. math-improvements Improve performance of long operations where one of the operands fits into an int. + diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -609,8 +609,14 @@ warnoptions.extend(pythonwarnings.split(',')) if warnoptions: sys.warnoptions[:] = warnoptions - from warnings import _processoptions - _processoptions(sys.warnoptions) + try: + if 'warnings' in sys.modules: + from warnings import _processoptions + _processoptions(sys.warnoptions) + else: + import warnings + except ImportError as e: + pass # CPython just eats any exception here # set up the Ctrl-C => KeyboardInterrupt signal handler, if the # signal module is available diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -977,6 +977,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -109,6 +109,7 @@ '_promote' : 'interp_magic._promote', 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -211,3 +211,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -644,6 +644,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', @@ -1180,6 +1181,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1726,6 +1731,12 @@ w_mod = state.fixup_extension(name, path) return w_mod +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -47,6 +47,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -261,6 +261,7 @@ if w_obj is not None: if w_obj is not w_marker_deallocating: return w_obj + type_name = rffi.charp2str(cts.cast('char*', ref.c_ob_type.c_tp_name)) fatalerror( "*** Invalid usage of a dying CPython object ***\n" "\n" @@ -275,7 +276,8 @@ "freed, making that reference point to garbage.\n" ">>> PyPy could contain some workaround to still work if\n" "you are lucky, but it is not done so far; better fix the bug in\n" - "the CPython extension.") + "the CPython extension.\n" + ">>> This object is of type '%s'" % (type_name,)) # This reference is not yet a real interpreter object. # Realize it. diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyInt_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -421,6 +421,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -366,10 +366,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -398,28 +402,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -211,9 +211,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -576,6 +576,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in e.value.message + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in excinfo.value.message - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in e.value.message +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in excinfo.value.message - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) From pypy.commits at gmail.com Sat Dec 15 16:15:54 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 15 Dec 2018 13:15:54 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5c156f0a.1c69fb81.bc292.236b@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95499:cbf95a8a15f1 Date: 2018-12-15 23:12 +0200 http://bitbucket.org/pypy/pypy/changeset/cbf95a8a15f1/ Log: merge default into branch diff too long, truncating to 2000 out of 10025 lines diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -5,7 +5,7 @@ from cffi import recompiler from extra_tests.cffi_tests.udir import udir from extra_tests.cffi_tests.support import u, long -from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture, _verify try: import importlib @@ -36,7 +36,7 @@ # add '-Werror' to the existing 'extra_compile_args' flags kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + ['-Werror']) - return recompiler._verify(ffi, module_name, source, *args, **kwds) + return _verify(ffi, module_name, source, *args, **kwds) def test_set_source_no_slashes(): ffi = FFI() @@ -1539,15 +1539,18 @@ assert (pt.x, pt.y) == (99*500*999, -99*500*999) def test_extern_python_1(): + import warnings ffi = FFI() - ffi.cdef(""" + with warnings.catch_warnings(record=True) as log: + ffi.cdef(""" extern "Python" { int bar(int, int); void baz(int, int); int bok(void); void boz(void); } - """) + """) + assert len(log) == 0, "got a warning: %r" % (log,) lib = verify(ffi, 'test_extern_python_1', """ static void baz(int, int); /* forward */ """) diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py --- a/extra_tests/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -4,6 +4,7 @@ from cffi import CDefError from cffi import recompiler from extra_tests.cffi_tests.support import * +from extra_tests.cffi_tests.support import _verify import _cffi_backend lib_m = ['m'] @@ -38,9 +39,8 @@ except AttributeError: pass self.set_source(module_name, preamble) - return recompiler._verify(self, module_name, preamble, *args, - extra_compile_args=self._extra_compile_args, - **kwds) + return _verify(self, module_name, preamble, *args, + extra_compile_args=self._extra_compile_args, **kwds) class FFI_warnings_not_error(FFI): _extra_compile_args = [] diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py --- a/extra_tests/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -62,3 +62,28 @@ def getvalue(self): return self._value + +def _verify(ffi, module_name, preamble, *args, **kwds): + import imp + from cffi.recompiler import recompile + from .udir import udir + assert module_name not in sys.modules, "module name conflict: %r" % ( + module_name,) + kwds.setdefault('tmpdir', str(udir)) + outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) + module = imp.load_dynamic(module_name, outputfilename) + # + # hack hack hack: copy all *bound methods* from module.ffi back to the + # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). + for name in dir(module.ffi): + if not name.startswith('_'): + attr = getattr(module.ffi, name) + if attr is not getattr(ffi, name, object()): + setattr(ffi, name, attr) + def typeof_disabled(*args, **kwds): + raise NotImplementedError + ffi._typeof = typeof_disabled + for name in dir(ffi): + if not name.startswith('_') and not hasattr(module.ffi, name): + setattr(ffi, name, NotImplemented) + return module.lib diff --git a/pypy/module/test_lib_pypy/ctypes_tests/__init__.py b/extra_tests/ctypes_tests/__init__.py rename from pypy/module/test_lib_pypy/ctypes_tests/__init__.py rename to extra_tests/ctypes_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/extra_tests/ctypes_tests/_ctypes_test.c rename from pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c rename to extra_tests/ctypes_tests/_ctypes_test.c diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py rename from pypy/module/test_lib_pypy/ctypes_tests/conftest.py rename to extra_tests/ctypes_tests/conftest.py --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -3,10 +3,6 @@ import sys import os -def pytest_ignore_collect(path): - if '__pypy__' not in sys.builtin_module_names: - return True - # XXX: copied from pypy/tool/cpyext/extbuild.py if os.name != 'nt': so_ext = 'so' @@ -85,8 +81,7 @@ return outputfilename # end copy -def compile_so_file(): - udir = pytest.ensuretemp('_ctypes_test') +def compile_so_file(udir): cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': @@ -96,8 +91,12 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) -# we need to run after the "tmpdir" plugin which installs pytest.ensuretemp - at pytest.mark.trylast -def pytest_configure(config): - global sofile - sofile = compile_so_file() + at pytest.fixture(scope='session') +def sofile(tmpdir_factory): + udir = tmpdir_factory.mktemp('_ctypes_test') + return str(compile_so_file(udir)) + + at pytest.fixture +def dll(sofile): + from ctypes import CDLL + return CDLL(str(sofile)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/extra_tests/ctypes_tests/support.py rename from pypy/module/test_lib_pypy/ctypes_tests/support.py rename to extra_tests/ctypes_tests/support.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_anon.py rename to extra_tests/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/extra_tests/ctypes_tests/test_anon.py @@ -1,86 +1,55 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestAnon(BaseCTypesTestChecker): + at pytest.mark.pypy_only +def test_nested(): + class ANON_S(Structure): + _fields_ = [("a", c_int)] - def test_anon(self): - class ANON(Union): - _fields_ = [("a", c_int), - ("b", c_int)] + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON), - ("y", c_int)] - _anonymous_ = ["_"] + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) + assert Y.x.offset == 0 + assert Y.a.offset == sizeof(c_int) + assert Y.b.offset == sizeof(c_int) + assert Y._.offset == sizeof(c_int) + assert Y.y.offset == sizeof(c_int) * 2 - assert ANON.a.offset == 0 - assert ANON.b.offset == 0 + assert Y._names_ == ['x', 'a', 'b', 'y'] - def test_anon_nonseq(self): - # TypeError: _anonymous_ must be a sequence - with pytest.raises(TypeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": 42}) +def test_anonymous_fields_on_instance(): + # this is about the *instance-level* access of anonymous fields, + # which you'd guess is the most common, but used not to work + # (issue #2230) - def test_anon_nonmember(self): - # AttributeError: type object 'Name' has no attribute 'x' - with pytest.raises(AttributeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": ["x"]}) + class B(Structure): + _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] + class A(Structure): + _anonymous_ = ["b"] + _fields_ = [("b", B)] - def test_nested(self): - class ANON_S(Structure): - _fields_ = [("a", c_int)] + a = A() + a.x = 5 + assert a.x == 5 + assert a.b.x == 5 + a.b.x += 1 + assert a.x == 6 - class ANON_U(Union): - _fields_ = [("_", ANON_S), - ("b", c_int)] - _anonymous_ = ["_"] + class C(Structure): + _anonymous_ = ["a"] + _fields_ = [("v", c_int), ("a", A)] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON_U), - ("y", c_int)] - _anonymous_ = ["_"] - - assert Y.x.offset == 0 - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) - assert Y._.offset == sizeof(c_int) - assert Y.y.offset == sizeof(c_int) * 2 - - assert Y._names_ == ['x', 'a', 'b', 'y'] - - def test_anonymous_fields_on_instance(self): - # this is about the *instance-level* access of anonymous fields, - # which you'd guess is the most common, but used not to work - # (issue #2230) - - class B(Structure): - _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] - class A(Structure): - _anonymous_ = ["b"] - _fields_ = [("b", B)] - - a = A() - a.x = 5 - assert a.x == 5 - assert a.b.x == 5 - a.b.x += 1 - assert a.x == 6 - - class C(Structure): - _anonymous_ = ["a"] - _fields_ = [("v", c_int), ("a", A)] - - c = C() - c.v = 3 - c.y = -8 - assert c.v == 3 - assert c.y == c.a.y == c.a.b.y == -8 - assert not hasattr(c, 'b') + c = C() + c.v = 3 + c.y = -8 + assert c.v == 3 + assert c.y == c.a.y == c.a.b.y == -8 + assert not hasattr(c, 'b') diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_array.py rename to extra_tests/ctypes_tests/test_array.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py +++ b/extra_tests/ctypes_tests/test_array.py @@ -1,177 +1,64 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -formats = "bBhHiIlLqQfd" +def test_slice(): + values = list(range(5)) + numarray = c_int * 5 -formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ - c_long, c_ulonglong, c_float, c_double + na = numarray(*(c_int(x) for x in values)) -class TestArray(BaseCTypesTestChecker): - def test_simple(self): - # create classes holding simple numeric types, and check - # various properties. + assert list(na[0:0]) == [] + assert list(na[:]) == values + assert list(na[:10]) == values - init = range(15, 25) +def test_init_again(): + sz = (c_char * 3)() + addr1 = addressof(sz) + sz.__init__(*b"foo") + addr2 = addressof(sz) + assert addr1 == addr2 - for fmt in formats: - alen = len(init) - int_array = ARRAY(fmt, alen) +def test_array_of_structures(): + class X(Structure): + _fields_ = [('x', c_int), ('y', c_int)] - ia = int_array(*init) - # length of instance ok? - assert len(ia) == alen + Y = X * 2 + y = Y() + x = X() + x.y = 3 + y[1] = x + assert y[1].y == 3 - # slot values ok? - values = [ia[i] for i in range(len(init))] - assert values == init +def test_output_simple(): + A = c_char * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # change the items - from operator import setitem - new_values = range(42, 42+alen) - [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(len(init))] - assert values == new_values + A = c_wchar * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # are the items initialized to 0? - ia = int_array() - values = [ia[i] for i in range(len(init))] - assert values == [0] * len(init) +def test_output_simple_array(): + A = c_char * 10 + AA = A * 10 + aa = AA() + assert aa[0] != b'' - # Too many in itializers should be caught - with pytest.raises(IndexError): - int_array(*range(alen*2)) +def test_output_complex_test(): + class Car(Structure): + _fields_ = [("brand", c_char * 10), + ("speed", c_float), + ("owner", c_char * 10)] - CharArray = ARRAY(c_char, 3) + assert isinstance(Car(b"abcdefghi", 42.0, b"12345").brand, bytes) + assert Car(b"abcdefghi", 42.0, b"12345").brand == b"abcdefghi" + assert Car(b"abcdefghio", 42.0, b"12345").brand == b"abcdefghio" + with pytest.raises(ValueError): + Car(b"abcdefghiop", 42.0, b"12345") - ca = CharArray("a", "b", "c") - - # Should this work? It doesn't: - # CharArray("abc") - with pytest.raises(TypeError): - CharArray("abc") - - assert ca[0] == "a" - assert ca[1] == "b" - assert ca[2] == "c" - assert ca[-3] == "a" - assert ca[-2] == "b" - assert ca[-1] == "c" - - assert len(ca) == 3 - - # slicing is now supported, but not extended slicing (3-argument)! - from operator import getslice, delitem - with pytest.raises(TypeError): - getslice(ca, 0, 1, -1) - - # cannot delete items - with pytest.raises(TypeError): - delitem(ca, 0) - - def test_numeric_arrays(self): - - alen = 5 - - numarray = ARRAY(c_int, alen) - - na = numarray() - values = [na[i] for i in range(alen)] - assert values == [0] * alen - - na = numarray(*[c_int()] * alen) - values = [na[i] for i in range(alen)] - assert values == [0]*alen - - na = numarray(1, 2, 3, 4, 5) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - na = numarray(*map(c_int, (1, 2, 3, 4, 5))) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - def test_slice(self): - values = range(5) - numarray = c_int * 5 - - na = numarray(*(c_int(x) for x in values)) - - assert list(na[0:0]) == [] - assert list(na[:]) == values - assert list(na[:10]) == values - - def test_classcache(self): - assert not ARRAY(c_int, 3) is ARRAY(c_int, 4) - assert ARRAY(c_int, 3) is ARRAY(c_int, 3) - - def test_from_address(self): - # Failed with 0.9.8, reported by JUrner - p = create_string_buffer("foo") - sz = (c_char * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - - def test_init_again(self): - sz = (c_char * 3)() - addr1 = addressof(sz) - sz.__init__(*"foo") - addr2 = addressof(sz) - assert addr1 == addr2 - - try: - create_unicode_buffer - except NameError: - pass - else: - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - -class TestSophisticatedThings(BaseCTypesTestChecker): - def test_array_of_structures(self): - class X(Structure): - _fields_ = [('x', c_int), ('y', c_int)] - - Y = X * 2 - y = Y() - x = X() - x.y = 3 - y[1] = x - assert y[1].y == 3 - - def test_output_simple(self): - A = c_char * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - A = c_wchar * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - def test_output_simple_array(self): - A = c_char * 10 - AA = A * 10 - aa = AA() - assert aa[0] != '' - - def test_output_complex_test(self): - class Car(Structure): - _fields_ = [("brand", c_char * 10), - ("speed", c_float), - ("owner", c_char * 10)] - - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" - with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") - - A = Car._fields_[2][1] - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = Car._fields_[2][1] + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_base.py rename to extra_tests/ctypes_tests/test_base.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py +++ b/extra_tests/ctypes_tests/test_base.py @@ -1,26 +1,24 @@ -from .support import WhiteBoxTests - +import pytest from ctypes import * -# WhiteBoxTests +pytestmark = pytest.mark.pypy_only -class TestCTypesBase(WhiteBoxTests): - def test_pointer(self): - p = pointer(pointer(c_int(2))) - x = p[0] - assert x._base is p +def test_pointer(): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p - def test_structure(self): - class X(Structure): - _fields_ = [('x', POINTER(c_int)), - ('y', POINTER(c_int))] +def test_structure(): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] - x = X() - assert x.y._base is x - assert x.y._index == 1 + x = X() + assert x.y._base is x + assert x.y._index == 1 - def test_array(self): - X = POINTER(c_int) * 24 - x = X() - assert x[16]._base is x - assert x[16]._index == 16 +def test_array(): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/extra_tests/ctypes_tests/test_bitfields.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py rename to extra_tests/ctypes_tests/test_bitfields.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py +++ b/extra_tests/ctypes_tests/test_bitfields.py @@ -1,249 +1,19 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -import os -import ctypes -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types +def test_set_fields_attr(): + class A(Structure): + pass + A._fields_ = [("a", c_byte), ("b", c_ubyte)] +def test_set_fields_attr_bitfields(): + class A(Structure): + pass + A._fields_ = [("a", POINTER(A)), ("b", c_ubyte, 4)] -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - func = CDLL(_ctypes_test).unpack_bitfields - func.argtypes = POINTER(BITS), c_char - mod.func = func - - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - - -class TestC: - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - def test_shorts(self): - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - -class TestBitField: - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - x.a, x.b, x.c = -1, 7, -1 - assert (x.a, x.b, x.c) == (-1, 7, -1) - - x = X() - x.a, x.b, x.c = -1, -7, -1 - assert (x.a, x.b, x.c) == (-1, -7, -1) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - assert (x.a, x.b, x.c) == (0, 0, 0) - x.a, x.b, x.c = 7, 2305843009213693953, 7 - assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ)*2 - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, -1, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, -1, 0) - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 7, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 7, 0) - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char_p') - - result = self.fail_fields(("a", c_void_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_void_p') - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - assert result == (TypeError, 'bit fields not allowed for type LP_c_int') - - result = self.fail_fields(("a", c_char, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char') - - try: - c_wchar - except NameError: - pass - else: - result = self.fail_fields(("a", c_wchar, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_wchar') - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - assert result == (TypeError, 'bit fields not allowed for type Dummy') - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - result = self.fail_fields(("a", c_typ, 0)) - assert result == (ValueError, 'number of bits invalid for bit field') - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - assert sizeof(X) == sizeof(c_typ) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == 0 - assert X.a1.offset == sizeof(c_short) - assert X.b.offset == sizeof(c_short)*2 - assert X.c.offset == sizeof(c_short)*2 - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == sizeof(c_short)*0 - assert X.b.offset == sizeof(c_short)*1 - assert X.c.offset == sizeof(c_short)*2 - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - import traceback - traceback.print_exc() - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name in ("nt", "ce"): - assert sizeof(X) == sizeof(c_int)*2 - else: - assert sizeof(X) == sizeof(c_int) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - assert sizeof(X) == sizeof(c_int)*2 - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - assert sizeof(X) == sizeof(c_byte) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - def test_set_fields_attr(self): - class A(Structure): - pass - A._fields_ = [("a", c_byte), - ("b", c_ubyte)] - - def test_set_fields_attr_bitfields(self): - class A(Structure): - pass - A._fields_ = [("a", POINTER(A)), - ("b", c_ubyte, 4)] - - def test_set_fields_cycle_fails(self): - class A(Structure): - pass - with pytest.raises(AttributeError): - A._fields_ = [("a", A)] +def test_set_fields_cycle_fails(): + class A(Structure): + pass + with pytest.raises(AttributeError): + A._fields_ = [("a", A)] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py rename to extra_tests/ctypes_tests/test_buffers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py +++ b/extra_tests/ctypes_tests/test_buffers.py @@ -1,76 +1,38 @@ from ctypes import * -from .support import BaseCTypesTestChecker -class TestStringBuffer(BaseCTypesTestChecker): +def test_buffer(): + b = create_string_buffer(32) + assert len(b) == 32 + assert sizeof(b) == 32 * sizeof(c_char) + assert type(b[0]) is str - def test_buffer(self): - b = create_string_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(33L) + assert len(b) == 33 + assert sizeof(b) == 33 * sizeof(c_char) + assert type(b[0]) is str - b = create_string_buffer(33L) - assert len(b) == 33 - assert sizeof(b) == 33 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(b"abc") + assert len(b) == 4 # trailing nul char + assert sizeof(b) == 4 * sizeof(c_char) + assert type(b[0]) is str + assert b[0] == b"a" + assert b[:] == b"abc\0" - b = create_string_buffer("abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" +def test_from_buffer(): + b1 = bytearray(b"abcde") + b = (c_char * 5).from_buffer(b1) + assert b[2] == b"c" + # + b1 = bytearray(b"abcd") + b = c_int.from_buffer(b1) + assert b.value in (1684234849, # little endian + 1633837924) # big endian - def test_string_conversion(self): - b = create_string_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" - - def test_from_buffer(self): - b1 = bytearray("abcde") - b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" - # - b1 = bytearray("abcd") - b = c_int.from_buffer(b1) - assert b.value in (1684234849, # little endian - 1633837924) # big endian - - def test_from_buffer_keepalive(self): - # Issue #2878 - b1 = bytearray("ab") - array = (c_uint16 * 32)() - array[6] = c_uint16.from_buffer(b1) - # this is also what we get on CPython. I don't think it makes - # sense because the array contains just a copy of the number. - assert array._objects == {'6': b1} - - try: - c_wchar - except NameError: - pass - else: - def test_unicode_buffer(self): - b = create_unicode_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_wchar) - assert type(b[0]) is unicode - - b = create_unicode_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - - def test_unicode_conversion(self): - b = create_unicode_buffer("abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - +def test_from_buffer_keepalive(): + # Issue #2878 + b1 = bytearray(b"ab") + array = (c_uint16 * 32)() + array[6] = c_uint16.from_buffer(b1) + # this is also what we get on CPython. I don't think it makes + # sense because the array contains just a copy of the number. + assert array._objects == {'6': b1} diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/extra_tests/ctypes_tests/test_callback_traceback.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py rename to extra_tests/ctypes_tests/test_callback_traceback.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py +++ b/extra_tests/ctypes_tests/test_callback_traceback.py @@ -1,80 +1,35 @@ # derived from test_random_things.py -import py +import pytest + from ctypes import * -import sys -def callback_func(arg): - 42 / arg - raise ValueError(arg) +_rawffi = pytest.importorskip('_rawffi') -class TestCallbackTraceback: - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. +# +# This test makes sure the exception types *and* the exception +# value is printed correctly. + + at pytest.mark.skipif("sys.flags.inspect") +def test_SystemExit(monkeypatch, capsys): + """ + When an exception is raised in a ctypes callback function, the C + code prints a traceback. When SystemExit is raised, the interpreter + normally exits immediately. + """ + def callback_func(arg): + raise SystemExit(42) + def custom_exit(value): + raise Exception("<<>>" % (value,)) + monkeypatch.setattr(_rawffi, 'exit', custom_exit) + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) + out, err = capsys.readouterr() + assert not err + cb2(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead a additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import StringIO - old_stderr = sys.stderr - logger = sys.stderr = StringIO.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() - - def test_ValueError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - assert out.splitlines()[-1] == ( - "ValueError: 42") - - def test_IntegerDivisionError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_FloatDivisionError(self): - cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_TypeErrorDivisionError(self): - cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, "spam") - assert out.splitlines()[-1].startswith( - "TypeError: " - "unsupported operand type(s) for") - - def test_SystemExit(self): - import _rawffi - if sys.flags.inspect: - skip("requires sys.flags.inspect == 0") - def callback_func(arg): - raise SystemExit(42) - def custom_exit(value): - raise Exception("<<>>" % (value,)) - original_exit = _rawffi.exit - try: - _rawffi.exit = custom_exit - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) - out = self.capture_stderr(cb2, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - finally: - _rawffi.exit = original_exit + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py rename to extra_tests/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/extra_tests/ctypes_tests/test_callbacks.py @@ -1,283 +1,194 @@ +import pytest + +import math from ctypes import * -import pytest from .support import BaseCTypesTestChecker -class TestCallbacks(BaseCTypesTestChecker): - functype = CFUNCTYPE - -## def tearDown(self): -## import gc -## gc.collect() - - def callback(self, *args): - self.got_args = args - return args[-1] - - def check_type(self, typ, arg): - unwrapped_types = { - c_float: (float,), - c_double: (float,), - c_char: (str,), - c_char_p: (str,), - c_uint: (int, long), - c_ulong: (int, long), - } - - PROTO = self.functype.im_func(typ, typ) - cfunc = PROTO(self.callback) - result = cfunc(arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (arg,) - assert result == arg - - result2 = cfunc(typ(arg)) - assert type(result2) in unwrapped_types.get(typ, (int, long)) - - PROTO = self.functype.im_func(typ, c_byte, typ) - result = PROTO(self.callback)(-3, arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (-3, arg) - assert result == arg - - ################ - - def test_byte(self): - self.check_type(c_byte, 42) - self.check_type(c_byte, -42) - - def test_ubyte(self): - self.check_type(c_ubyte, 42) - - def test_short(self): - self.check_type(c_short, 42) - self.check_type(c_short, -42) - - def test_ushort(self): - self.check_type(c_ushort, 42) - - def test_int(self): - self.check_type(c_int, 42) - self.check_type(c_int, -42) - - def test_uint(self): - self.check_type(c_uint, 42) - - def test_long(self): - self.check_type(c_long, 42) - self.check_type(c_long, -42) - - def test_ulong(self): - self.check_type(c_ulong, 42) - - def test_longlong(self): - self.check_type(c_longlong, 42) - self.check_type(c_longlong, -42) - - def test_ulonglong(self): - self.check_type(c_ulonglong, 42) - - def test_float(self): - # only almost equal: double -> float -> double - import math - self.check_type(c_float, math.e) - self.check_type(c_float, -math.e) - - def test_double(self): - self.check_type(c_double, 3.14) - self.check_type(c_double, -3.14) - - def test_char(self): - self.check_type(c_char, "x") - self.check_type(c_char, "a") - - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. -## def test_char_p(self): -## self.check_type(c_char_p, "abc") -## self.check_type(c_char_p, "def") - - - @pytest.mark.xfail( - reason="we are less strict about callback return type sanity") - def test_unsupported_restype_1(self): - # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. - # POINTER(c_double), for example, is not supported. - - prototype = self.functype.im_func(POINTER(c_double)) - # The type is checked when the prototype is called - with pytest.raises(TypeError): - prototype(lambda: None) - +functypes = [CFUNCTYPE] try: - WINFUNCTYPE + functypes.append(WINFUNCTYPE) except NameError: pass -else: - class TestStdcallCallbacks(TestCallbacks): - functype = WINFUNCTYPE -################################################################ -class TestSampleCallbacks(BaseCTypesTestChecker): +def callback(*args): + callback.got_args = args + return args[-1] - def test_integrate(self): - # Derived from some then non-working code, posted by David Foster - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) +unwrapped_types = { + c_float: (float,), + c_double: (float,), + c_char: (str,), + c_char_p: (str,), + c_uint: (int, long), + c_ulong: (int, long), + } - # The function prototype called by 'integrate': double func(double); - CALLBACK = CFUNCTYPE(c_double, c_double) + at pytest.mark.parametrize("typ, arg", [ + (c_byte, 42), + (c_byte, -42), + (c_ubyte, 42), + (c_short, 42), + (c_short, -42), + (c_ushort, 42), + (c_int, 42), + (c_int, -42), + (c_uint, 42), + (c_long, 42), + (c_long, -42), + (c_ulong, 42), + (c_longlong, 42), + (c_longlong, -42), + (c_ulonglong, 42), + (c_float, math.e), # only almost equal: double -> float -> double + (c_float, -math.e), + (c_double, 3.14), + (c_double, -3.14), + (c_char, b"x"), + (c_char, b"a"), +]) + at pytest.mark.parametrize('functype', functypes) +def test_types(typ, arg, functype): + PROTO = functype(typ, typ) + cfunc = PROTO(callback) + result = cfunc(arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (arg,) + assert result == arg - # The integrate function itself, exposed from the _ctypes_test dll - integrate = dll.integrate - integrate.argtypes = (c_double, c_double, CALLBACK, c_long) - integrate.restype = c_double + result2 = cfunc(typ(arg)) + assert type(result2) in unwrapped_types.get(typ, (int, long)) - def func(x): - print 'calculating x**2 of',x - return x**2 + PROTO = functype(typ, c_byte, typ) + result = PROTO(callback)(-3, arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (-3, arg) + assert result == arg - result = integrate(0.0, 1.0, CALLBACK(func), 10) - diff = abs(result - 1./3.) + at pytest.mark.parametrize('functype', functypes) +def test_unsupported_restype_1(functype): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. - assert diff < 0.01, "%s not less than 0.01" % diff + prototype = functype(POINTER(c_double)) + # The type is checked when the prototype is called + with pytest.raises(TypeError): + prototype(lambda: None) -################################################################ -class TestMoreCallbacks(BaseCTypesTestChecker): +def test_callback_with_struct_argument(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - def test_callback_with_struct_argument(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] + proto = CFUNCTYPE(c_int, RECT) - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - point.left *= -1 - return point.left+point.top+point.right+point.bottom + def callback(point): + point.left *= -1 + return point.left + point.top + point.right + point.bottom - cbp = proto(callback) + cbp = proto(callback) + rect = RECT(-1000, 100, 10, 1) + res = cbp(rect) + assert res == 1111 + assert rect.left == -1000 # must not have been changed! - rect = RECT(-1000,100,10,1) +def test_callback_from_c_with_struct_argument(dll): + class RECT(Structure): + _fields_ = [("left", c_long), ("top", c_long), + ("right", c_long), ("bottom", c_long)] - res = cbp(rect) + proto = CFUNCTYPE(c_int, RECT) - assert res == 1111 - assert rect.left == -1000 # must not have been changed! + def callback(point): + return point.left + point.top + point.right + point.bottom - def test_callback_from_c_with_struct_argument(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + cbp = proto(callback) + rect = RECT(1000, 100, 10, 1) - class RECT(Structure): - _fields_ = [("left", c_long), ("top", c_long), - ("right", c_long), ("bottom", c_long)] + call_callback_with_rect = dll.call_callback_with_rect + call_callback_with_rect.restype = c_int + call_callback_with_rect.argtypes = [proto, RECT] + res = call_callback_with_rect(cbp, rect) + assert res == 1111 - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - return point.left+point.top+point.right+point.bottom +def test_callback_unsupported_return_struct(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - cbp = proto(callback) - rect = RECT(1000,100,10,1) + proto = CFUNCTYPE(RECT, c_int) + with pytest.raises(TypeError): + proto(lambda r: 0) - call_callback_with_rect = dll.call_callback_with_rect - call_callback_with_rect.restype = c_int - call_callback_with_rect.argtypes = [proto, RECT] - res = call_callback_with_rect(cbp, rect) - assert res == 1111 - def test_callback_unsupported_return_struct(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] +def test_qsort(dll): + PI = POINTER(c_int) + A = c_int*5 + a = A() + for i in range(5): + a[i] = 5-i - proto = CFUNCTYPE(RECT, c_int) - with pytest.raises(TypeError): - proto(lambda r: 0) + assert a[0] == 5 # sanity + def comp(a, b): + a = a.contents.value + b = b.contents.value + return cmp(a,b) + qs = dll.my_qsort + qs.restype = None + CMP = CFUNCTYPE(c_int, PI, PI) + qs.argtypes = (PI, c_size_t, c_size_t, CMP) - def test_qsort(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) - PI = POINTER(c_int) - A = c_int*5 - a = A() - for i in range(5): - a[i] = 5-i + res = list(a) - assert a[0] == 5 # sanity + assert res == [1,2,3,4,5] - def comp(a, b): - a = a.contents.value - b = b.contents.value - return cmp(a,b) - qs = dll.my_qsort - qs.restype = None - CMP = CFUNCTYPE(c_int, PI, PI) - qs.argtypes = (PI, c_size_t, c_size_t, CMP) +def test_pyobject_as_opaque(dll): + def callback(arg): + return arg() - qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) + CTP = CFUNCTYPE(c_int, py_object) + cfunc = dll._testfunc_callback_opaque + cfunc.argtypes = [CTP, py_object] + cfunc.restype = c_int + res = cfunc(CTP(callback), lambda : 3) + assert res == 3 - res = list(a) +def test_callback_void(capsys, dll): + def callback(): + pass - assert res == [1,2,3,4,5] + CTP = CFUNCTYPE(None) + cfunc = dll._testfunc_callback_void + cfunc.argtypes = [CTP] + cfunc.restype = int + cfunc(CTP(callback)) + out, err = capsys.readouterr() + assert (out, err) == ("", "") - def test_pyobject_as_opaque(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - def callback(arg): - return arg() +def test_callback_pyobject(): + def callback(obj): + return obj - CTP = CFUNCTYPE(c_int, py_object) - cfunc = dll._testfunc_callback_opaque - cfunc.argtypes = [CTP, py_object] - cfunc.restype = c_int - res = cfunc(CTP(callback), lambda : 3) - assert res == 3 + FUNC = CFUNCTYPE(py_object, py_object) + cfunc = FUNC(callback) + param = c_int(42) + assert cfunc(param) is param - def test_callback_void(self, capsys): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - - def callback(): - pass - - CTP = CFUNCTYPE(None) - cfunc = dll._testfunc_callback_void - cfunc.argtypes = [CTP] - cfunc.restype = int - cfunc(CTP(callback)) - out, err = capsys.readouterr() - assert (out, err) == ("", "") - - - def test_callback_pyobject(self): - def callback(obj): - return obj - - FUNC = CFUNCTYPE(py_object, py_object) - cfunc = FUNC(callback) - param = c_int(42) - assert cfunc(param) is param - - def test_raise_argumenterror(self): - def callback(x): - pass - FUNC = CFUNCTYPE(None, c_void_p) - cfunc = FUNC(callback) - param = c_uint(42) - with pytest.raises(ArgumentError): - cfunc(param) +def test_raise_argumenterror(): + def callback(x): + pass + FUNC = CFUNCTYPE(None, c_void_p) + cfunc = FUNC(callback) + param = c_uint(42) + with pytest.raises(ArgumentError): + cfunc(param) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/extra_tests/ctypes_tests/test_cast.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_cast.py rename to extra_tests/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/extra_tests/ctypes_tests/test_cast.py @@ -1,106 +1,30 @@ +import pytest + from ctypes import * -import sys, py -from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - mod.lib = CDLL(str(conftest.sofile)) +def test_cast_functype(dll): + # make sure we can cast function type + my_sqrt = dll.my_sqrt + saved_objects = my_sqrt._objects.copy() + sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) + assert sqrt(4.0) == 2.0 + assert not cast(0, CFUNCTYPE(c_int)) + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) -class TestCast(BaseCTypesTestChecker): +def test_cast_argumenterror(): + param = c_uint(42) + with pytest.raises(ArgumentError): + cast(param, c_void_p) - def test_array2pointer(self): - array = (c_int * 3)(42, 17, 2) - - # casting an array to a pointer works. - ptr = cast(array, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - if 2*sizeof(c_short) == sizeof(c_int): - ptr = cast(array, POINTER(c_short)) - if sys.byteorder == "little": - assert [ptr[i] for i in range(6)] == ( - [42, 0, 17, 0, 2, 0]) - else: - assert [ptr[i] for i in range(6)] == ( - [0, 42, 0, 17, 0, 2]) - - def test_address2pointer(self): - array = (c_int * 3)(42, 17, 2) - - address = addressof(array) - ptr = cast(c_void_p(address), POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - ptr = cast(address, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - def test_p2a_objects(self): - py.test.skip("we make copies of strings") - array = (c_char_p * 5)() - assert array._objects is None - array[0] = "foo bar" - assert array._objects == {'0': "foo bar"} - - p = cast(array, POINTER(c_char_p)) - # array and p share a common _objects attribute - assert p._objects is array._objects - assert array._objects == {'0': "foo bar", id(array): array} - p[0] = "spam spam" - assert p._objects == {'0': "spam spam", id(array): array} - assert array._objects is p._objects - p[1] = "foo bar" - assert p._objects == {'1': 'foo bar', '0': "spam spam", id(array): array} - assert array._objects is p._objects - - def test_other(self): - p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) - assert p[:4] == [1,2, 3, 4] - c_int() - assert p[:4] == [1, 2, 3, 4] - p[2] = 96 - assert p[:4] == [1, 2, 96, 4] - c_int() - assert p[:4] == [1, 2, 96, 4] - - def test_char_p(self): - # This didn't work: bad argument to internal function - s = c_char_p("hiho") - - assert cast(cast(s, c_void_p), c_char_p).value == ( - "hiho") - - try: - c_wchar_p - except NameError: - pass - else: - def test_wchar_p(self): - s = c_wchar_p("hiho") - assert cast(cast(s, c_void_p), c_wchar_p).value == ( - "hiho") - - def test_cast_functype(self): - # make sure we can cast function type - my_sqrt = lib.my_sqrt - saved_objects = my_sqrt._objects.copy() - sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) - assert sqrt(4.0) == 2.0 - assert not cast(0, CFUNCTYPE(c_int)) - # - assert sqrt._objects is my_sqrt._objects # on CPython too - my_sqrt._objects.clear() - my_sqrt._objects.update(saved_objects) - - def test_cast_argumenterror(self): - param = c_uint(42) - py.test.raises(ArgumentError, "cast(param, c_void_p)") - - def test_c_bool(self): - x = c_bool(42) - assert x.value is True - x = c_bool(0.0) - assert x.value is False - x = c_bool("") - assert x.value is False - x = c_bool(['yadda']) - assert x.value is True +def test_c_bool(): + x = c_bool(42) + assert x.value is True + x = c_bool(0.0) + assert x.value is False + x = c_bool("") + assert x.value is False + x = c_bool(['yadda']) + assert x.value is True diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py b/extra_tests/ctypes_tests/test_commethods.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py rename to extra_tests/ctypes_tests/test_commethods.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/extra_tests/ctypes_tests/test_errno.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_errno.py rename to extra_tests/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/extra_tests/ctypes_tests/test_errno.py @@ -1,26 +1,19 @@ -import py +import pytest import ctypes -from _ctypes import function +_rawffi = pytest.importorskip('_rawffi') # PyPy-only -try: - import _rawffi -except ImportError: - py.test.skip("app-level test only for PyPy") +def test_errno_saved_and_restored(): + def check(): + assert _rawffi.get_errno() == 42 + assert ctypes.get_errno() == old + check.free_temp_buffers = lambda *args: None + f = ctypes._CFuncPtr() + old = _rawffi.get_errno() + f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO + ctypes.set_errno(42) + f._call_funcptr(check) + assert _rawffi.get_errno() == old + ctypes.set_errno(0) -class TestErrno: - - def test_errno_saved_and_restored(self): - def check(): - assert _rawffi.get_errno() == 42 - assert ctypes.get_errno() == old - check.free_temp_buffers = lambda *args: None - f = function.CFuncPtr() - old = _rawffi.get_errno() - f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO - ctypes.set_errno(42) - f._call_funcptr(check) - assert _rawffi.get_errno() == old - ctypes.set_errno(0) - - # see also test_functions.test_errno +# see also test_functions.test_errno diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_extra.py rename to extra_tests/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/extra_tests/ctypes_tests/test_extra.py @@ -5,244 +5,239 @@ import py from ctypes import * -from .support import BaseCTypesTestChecker -class TestExtra(BaseCTypesTestChecker): - def test_primitive_pointer(self): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 +def test_primitive_pointer(): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 - ## *** the rest is commented out because it should work but occasionally - ## *** trigger a ctypes bug (SourceForge bug #1467852). *** - ## q = pointer(y) - ## pp[0] = q # pp ---> p ---> y = 14 - ## assert y.value == 14 # (^^^ not q! ) - ## assert p.contents.value == 14 - ## assert pp.contents.contents.value == 14 - ## q.contents = x - ## assert pp.contents.contents.value == 14 +## *** the rest is commented out because it should work but occasionally +## *** trigger a ctypes bug (SourceForge bug #1467852). *** +## q = pointer(y) +## pp[0] = q # pp ---> p ---> y = 14 +## assert y.value == 14 # (^^^ not q! ) +## assert p.contents.value == 14 +## assert pp.contents.contents.value == 14 +## q.contents = x +## assert pp.contents.contents.value == 14 - def test_char_p(self): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" +def test_char_p(): + x = c_char_p(b"hello\x00world") + assert x.value == b"hello" + x.value = b"world" + assert x.value == b"world" - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + p = pointer(x) + assert p[0] == x.value == b"world" + p[0] = b"other" + assert x.value == p.contents.value == p[0] == b"other" - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + myarray = (c_char_p * 10)() + myarray[7] = b"hello" + assert isinstance(myarray[7], str) + assert myarray[7] == b"hello" - def test_struct(self): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] +def test_struct(): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 - def test_ptr_array(self): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) +def test_ptr_array(): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 - def test_void_p(self): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 +def test_void_p(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 - def test_char_array(self): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' +def test_char_array(): + a = (c_char * 3)() + a[0] = b'x' + a[1] = b'y' + assert a.value == b'xy' + a[2] = b'z' + assert a.value == b'xyz' - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = b"nxw" + assert b[0] == b'n' + assert b[1] == b'x' + assert b[2] == b'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = b"?" + assert b[0] == b'?' + assert b[1] == b'\x00' + assert b[2] == b'w' - class S(Structure): - _fields_ = [('p', POINTER(c_char))] + class S(Structure): + _fields_ = [('p', POINTER(c_char))] - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s = S() + s.p = b + s.p.contents.value = b'!' + assert b.value == b'!' - assert len(create_string_buffer(0)) == 0 + assert len(create_string_buffer(0)) == 0 - def test_array(self): - a = (c_int * 10)() +def test_array(): + a = (c_int * 10)() - class S(Structure): - _fields_ = [('p', POINTER(c_int))] + class S(Structure): + _fields_ = [('p', POINTER(c_int))] - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] - def test_truth_value(self): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... +def test_truth_value(): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) From pypy.commits at gmail.com Sat Dec 15 16:15:57 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 15 Dec 2018 13:15:57 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix translation Message-ID: <5c156f0d.1c69fb81.c4c42.03ae@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95500:fba36a06ee6e Date: 2018-12-15 23:13 +0200 http://bitbucket.org/pypy/pypy/changeset/fba36a06ee6e/ Log: fix translation diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -460,6 +460,7 @@ ptr = self.start_ptr if not self.next_char_ok(ctx, pattern, ptr, self.ppos3): return + assert not isinstance(ctx, AbstractMatchContext) self.start_ptr = ctx.next(ptr) return self.find_first_result(ctx, pattern) From pypy.commits at gmail.com Sat Dec 15 16:15:59 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 15 Dec 2018 13:15:59 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5c156f0f.1c69fb81.b3b97.ece5@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95501:fa5c55f5bebb Date: 2018-12-15 23:15 +0200 http://bitbucket.org/pypy/pypy/changeset/fa5c55f5bebb/ Log: merge py3.5 into branch diff too long, truncating to 2000 out of 9477 lines diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -5,7 +5,7 @@ from cffi import recompiler from extra_tests.cffi_tests.udir import udir from extra_tests.cffi_tests.support import u, long -from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture, _verify try: import importlib @@ -36,7 +36,7 @@ # add '-Werror' to the existing 'extra_compile_args' flags kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + ['-Werror']) - return recompiler._verify(ffi, module_name, source, *args, **kwds) + return _verify(ffi, module_name, source, *args, **kwds) def test_set_source_no_slashes(): ffi = FFI() @@ -1539,15 +1539,18 @@ assert (pt.x, pt.y) == (99*500*999, -99*500*999) def test_extern_python_1(): + import warnings ffi = FFI() - ffi.cdef(""" + with warnings.catch_warnings(record=True) as log: + ffi.cdef(""" extern "Python" { int bar(int, int); void baz(int, int); int bok(void); void boz(void); } - """) + """) + assert len(log) == 0, "got a warning: %r" % (log,) lib = verify(ffi, 'test_extern_python_1', """ static void baz(int, int); /* forward */ """) diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py --- a/extra_tests/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -4,6 +4,7 @@ from cffi import CDefError from cffi import recompiler from extra_tests.cffi_tests.support import * +from extra_tests.cffi_tests.support import _verify import _cffi_backend lib_m = ['m'] @@ -38,9 +39,8 @@ except AttributeError: pass self.set_source(module_name, preamble) - return recompiler._verify(self, module_name, preamble, *args, - extra_compile_args=self._extra_compile_args, - **kwds) + return _verify(self, module_name, preamble, *args, + extra_compile_args=self._extra_compile_args, **kwds) class FFI_warnings_not_error(FFI): _extra_compile_args = [] diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py --- a/extra_tests/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -62,3 +62,28 @@ def getvalue(self): return self._value + +def _verify(ffi, module_name, preamble, *args, **kwds): + import imp + from cffi.recompiler import recompile + from .udir import udir + assert module_name not in sys.modules, "module name conflict: %r" % ( + module_name,) + kwds.setdefault('tmpdir', str(udir)) + outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) + module = imp.load_dynamic(module_name, outputfilename) + # + # hack hack hack: copy all *bound methods* from module.ffi back to the + # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). + for name in dir(module.ffi): + if not name.startswith('_'): + attr = getattr(module.ffi, name) + if attr is not getattr(ffi, name, object()): + setattr(ffi, name, attr) + def typeof_disabled(*args, **kwds): + raise NotImplementedError + ffi._typeof = typeof_disabled + for name in dir(ffi): + if not name.startswith('_') and not hasattr(module.ffi, name): + setattr(ffi, name, NotImplemented) + return module.lib diff --git a/pypy/module/test_lib_pypy/ctypes_tests/__init__.py b/extra_tests/ctypes_tests/__init__.py rename from pypy/module/test_lib_pypy/ctypes_tests/__init__.py rename to extra_tests/ctypes_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/extra_tests/ctypes_tests/_ctypes_test.c rename from pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c rename to extra_tests/ctypes_tests/_ctypes_test.c diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py rename from pypy/module/test_lib_pypy/ctypes_tests/conftest.py rename to extra_tests/ctypes_tests/conftest.py --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -3,10 +3,6 @@ import sys import os -def pytest_ignore_collect(path): - if '__pypy__' not in sys.builtin_module_names: - return True - # XXX: copied from pypy/tool/cpyext/extbuild.py if os.name != 'nt': so_ext = 'so' @@ -85,8 +81,7 @@ return outputfilename # end copy -def compile_so_file(): - udir = pytest.ensuretemp('_ctypes_test') +def compile_so_file(udir): cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': @@ -96,8 +91,12 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) -# we need to run after the "tmpdir" plugin which installs pytest.ensuretemp - at pytest.mark.trylast -def pytest_configure(config): - global sofile - sofile = compile_so_file() + at pytest.fixture(scope='session') +def sofile(tmpdir_factory): + udir = tmpdir_factory.mktemp('_ctypes_test') + return str(compile_so_file(udir)) + + at pytest.fixture +def dll(sofile): + from ctypes import CDLL + return CDLL(str(sofile)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/extra_tests/ctypes_tests/support.py rename from pypy/module/test_lib_pypy/ctypes_tests/support.py rename to extra_tests/ctypes_tests/support.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_anon.py rename to extra_tests/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/extra_tests/ctypes_tests/test_anon.py @@ -1,86 +1,55 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestAnon(BaseCTypesTestChecker): + at pytest.mark.pypy_only +def test_nested(): + class ANON_S(Structure): + _fields_ = [("a", c_int)] - def test_anon(self): - class ANON(Union): - _fields_ = [("a", c_int), - ("b", c_int)] + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON), - ("y", c_int)] - _anonymous_ = ["_"] + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) + assert Y.x.offset == 0 + assert Y.a.offset == sizeof(c_int) + assert Y.b.offset == sizeof(c_int) + assert Y._.offset == sizeof(c_int) + assert Y.y.offset == sizeof(c_int) * 2 - assert ANON.a.offset == 0 - assert ANON.b.offset == 0 + assert Y._names_ == ['x', 'a', 'b', 'y'] - def test_anon_nonseq(self): - # TypeError: _anonymous_ must be a sequence - with pytest.raises(TypeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": 42}) +def test_anonymous_fields_on_instance(): + # this is about the *instance-level* access of anonymous fields, + # which you'd guess is the most common, but used not to work + # (issue #2230) - def test_anon_nonmember(self): - # AttributeError: type object 'Name' has no attribute 'x' - with pytest.raises(AttributeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": ["x"]}) + class B(Structure): + _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] + class A(Structure): + _anonymous_ = ["b"] + _fields_ = [("b", B)] - def test_nested(self): - class ANON_S(Structure): - _fields_ = [("a", c_int)] + a = A() + a.x = 5 + assert a.x == 5 + assert a.b.x == 5 + a.b.x += 1 + assert a.x == 6 - class ANON_U(Union): - _fields_ = [("_", ANON_S), - ("b", c_int)] - _anonymous_ = ["_"] + class C(Structure): + _anonymous_ = ["a"] + _fields_ = [("v", c_int), ("a", A)] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON_U), - ("y", c_int)] - _anonymous_ = ["_"] - - assert Y.x.offset == 0 - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) - assert Y._.offset == sizeof(c_int) - assert Y.y.offset == sizeof(c_int) * 2 - - assert Y._names_ == ['x', 'a', 'b', 'y'] - - def test_anonymous_fields_on_instance(self): - # this is about the *instance-level* access of anonymous fields, - # which you'd guess is the most common, but used not to work - # (issue #2230) - - class B(Structure): - _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] - class A(Structure): - _anonymous_ = ["b"] - _fields_ = [("b", B)] - - a = A() - a.x = 5 - assert a.x == 5 - assert a.b.x == 5 - a.b.x += 1 - assert a.x == 6 - - class C(Structure): - _anonymous_ = ["a"] - _fields_ = [("v", c_int), ("a", A)] - - c = C() - c.v = 3 - c.y = -8 - assert c.v == 3 - assert c.y == c.a.y == c.a.b.y == -8 - assert not hasattr(c, 'b') + c = C() + c.v = 3 + c.y = -8 + assert c.v == 3 + assert c.y == c.a.y == c.a.b.y == -8 + assert not hasattr(c, 'b') diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_array.py rename to extra_tests/ctypes_tests/test_array.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py +++ b/extra_tests/ctypes_tests/test_array.py @@ -1,177 +1,64 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -formats = "bBhHiIlLqQfd" +def test_slice(): + values = list(range(5)) + numarray = c_int * 5 -formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ - c_long, c_ulonglong, c_float, c_double + na = numarray(*(c_int(x) for x in values)) -class TestArray(BaseCTypesTestChecker): - def test_simple(self): - # create classes holding simple numeric types, and check - # various properties. + assert list(na[0:0]) == [] + assert list(na[:]) == values + assert list(na[:10]) == values - init = range(15, 25) +def test_init_again(): + sz = (c_char * 3)() + addr1 = addressof(sz) + sz.__init__(*b"foo") + addr2 = addressof(sz) + assert addr1 == addr2 - for fmt in formats: - alen = len(init) - int_array = ARRAY(fmt, alen) +def test_array_of_structures(): + class X(Structure): + _fields_ = [('x', c_int), ('y', c_int)] - ia = int_array(*init) - # length of instance ok? - assert len(ia) == alen + Y = X * 2 + y = Y() + x = X() + x.y = 3 + y[1] = x + assert y[1].y == 3 - # slot values ok? - values = [ia[i] for i in range(len(init))] - assert values == init +def test_output_simple(): + A = c_char * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # change the items - from operator import setitem - new_values = range(42, 42+alen) - [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(len(init))] - assert values == new_values + A = c_wchar * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # are the items initialized to 0? - ia = int_array() - values = [ia[i] for i in range(len(init))] - assert values == [0] * len(init) +def test_output_simple_array(): + A = c_char * 10 + AA = A * 10 + aa = AA() + assert aa[0] != b'' - # Too many in itializers should be caught - with pytest.raises(IndexError): - int_array(*range(alen*2)) +def test_output_complex_test(): + class Car(Structure): + _fields_ = [("brand", c_char * 10), + ("speed", c_float), + ("owner", c_char * 10)] - CharArray = ARRAY(c_char, 3) + assert isinstance(Car(b"abcdefghi", 42.0, b"12345").brand, bytes) + assert Car(b"abcdefghi", 42.0, b"12345").brand == b"abcdefghi" + assert Car(b"abcdefghio", 42.0, b"12345").brand == b"abcdefghio" + with pytest.raises(ValueError): + Car(b"abcdefghiop", 42.0, b"12345") - ca = CharArray("a", "b", "c") - - # Should this work? It doesn't: - # CharArray("abc") - with pytest.raises(TypeError): - CharArray("abc") - - assert ca[0] == "a" - assert ca[1] == "b" - assert ca[2] == "c" - assert ca[-3] == "a" - assert ca[-2] == "b" - assert ca[-1] == "c" - - assert len(ca) == 3 - - # slicing is now supported, but not extended slicing (3-argument)! - from operator import getslice, delitem - with pytest.raises(TypeError): - getslice(ca, 0, 1, -1) - - # cannot delete items - with pytest.raises(TypeError): - delitem(ca, 0) - - def test_numeric_arrays(self): - - alen = 5 - - numarray = ARRAY(c_int, alen) - - na = numarray() - values = [na[i] for i in range(alen)] - assert values == [0] * alen - - na = numarray(*[c_int()] * alen) - values = [na[i] for i in range(alen)] - assert values == [0]*alen - - na = numarray(1, 2, 3, 4, 5) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - na = numarray(*map(c_int, (1, 2, 3, 4, 5))) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - def test_slice(self): - values = range(5) - numarray = c_int * 5 - - na = numarray(*(c_int(x) for x in values)) - - assert list(na[0:0]) == [] - assert list(na[:]) == values - assert list(na[:10]) == values - - def test_classcache(self): - assert not ARRAY(c_int, 3) is ARRAY(c_int, 4) - assert ARRAY(c_int, 3) is ARRAY(c_int, 3) - - def test_from_address(self): - # Failed with 0.9.8, reported by JUrner - p = create_string_buffer("foo") - sz = (c_char * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - - def test_init_again(self): - sz = (c_char * 3)() - addr1 = addressof(sz) - sz.__init__(*"foo") - addr2 = addressof(sz) - assert addr1 == addr2 - - try: - create_unicode_buffer - except NameError: - pass - else: - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - -class TestSophisticatedThings(BaseCTypesTestChecker): - def test_array_of_structures(self): - class X(Structure): - _fields_ = [('x', c_int), ('y', c_int)] - - Y = X * 2 - y = Y() - x = X() - x.y = 3 - y[1] = x - assert y[1].y == 3 - - def test_output_simple(self): - A = c_char * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - A = c_wchar * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - def test_output_simple_array(self): - A = c_char * 10 - AA = A * 10 - aa = AA() - assert aa[0] != '' - - def test_output_complex_test(self): - class Car(Structure): - _fields_ = [("brand", c_char * 10), - ("speed", c_float), - ("owner", c_char * 10)] - - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" - with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") - - A = Car._fields_[2][1] - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = Car._fields_[2][1] + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_base.py rename to extra_tests/ctypes_tests/test_base.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py +++ b/extra_tests/ctypes_tests/test_base.py @@ -1,26 +1,24 @@ -from .support import WhiteBoxTests - +import pytest from ctypes import * -# WhiteBoxTests +pytestmark = pytest.mark.pypy_only -class TestCTypesBase(WhiteBoxTests): - def test_pointer(self): - p = pointer(pointer(c_int(2))) - x = p[0] - assert x._base is p +def test_pointer(): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p - def test_structure(self): - class X(Structure): - _fields_ = [('x', POINTER(c_int)), - ('y', POINTER(c_int))] +def test_structure(): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] - x = X() - assert x.y._base is x - assert x.y._index == 1 + x = X() + assert x.y._base is x + assert x.y._index == 1 - def test_array(self): - X = POINTER(c_int) * 24 - x = X() - assert x[16]._base is x - assert x[16]._index == 16 +def test_array(): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/extra_tests/ctypes_tests/test_bitfields.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py rename to extra_tests/ctypes_tests/test_bitfields.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py +++ b/extra_tests/ctypes_tests/test_bitfields.py @@ -1,249 +1,19 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -import os -import ctypes -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types +def test_set_fields_attr(): + class A(Structure): + pass + A._fields_ = [("a", c_byte), ("b", c_ubyte)] +def test_set_fields_attr_bitfields(): + class A(Structure): + pass + A._fields_ = [("a", POINTER(A)), ("b", c_ubyte, 4)] -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - func = CDLL(_ctypes_test).unpack_bitfields - func.argtypes = POINTER(BITS), c_char - mod.func = func - - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - - -class TestC: - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - def test_shorts(self): - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - -class TestBitField: - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - x.a, x.b, x.c = -1, 7, -1 - assert (x.a, x.b, x.c) == (-1, 7, -1) - - x = X() - x.a, x.b, x.c = -1, -7, -1 - assert (x.a, x.b, x.c) == (-1, -7, -1) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - assert (x.a, x.b, x.c) == (0, 0, 0) - x.a, x.b, x.c = 7, 2305843009213693953, 7 - assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ)*2 - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, -1, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, -1, 0) - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 7, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 7, 0) - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char_p') - - result = self.fail_fields(("a", c_void_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_void_p') - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - assert result == (TypeError, 'bit fields not allowed for type LP_c_int') - - result = self.fail_fields(("a", c_char, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char') - - try: - c_wchar - except NameError: - pass - else: - result = self.fail_fields(("a", c_wchar, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_wchar') - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - assert result == (TypeError, 'bit fields not allowed for type Dummy') - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - result = self.fail_fields(("a", c_typ, 0)) - assert result == (ValueError, 'number of bits invalid for bit field') - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - assert sizeof(X) == sizeof(c_typ) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == 0 - assert X.a1.offset == sizeof(c_short) - assert X.b.offset == sizeof(c_short)*2 - assert X.c.offset == sizeof(c_short)*2 - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == sizeof(c_short)*0 - assert X.b.offset == sizeof(c_short)*1 - assert X.c.offset == sizeof(c_short)*2 - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - import traceback - traceback.print_exc() - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name in ("nt", "ce"): - assert sizeof(X) == sizeof(c_int)*2 - else: - assert sizeof(X) == sizeof(c_int) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - assert sizeof(X) == sizeof(c_int)*2 - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - assert sizeof(X) == sizeof(c_byte) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - def test_set_fields_attr(self): - class A(Structure): - pass - A._fields_ = [("a", c_byte), - ("b", c_ubyte)] - - def test_set_fields_attr_bitfields(self): - class A(Structure): - pass - A._fields_ = [("a", POINTER(A)), - ("b", c_ubyte, 4)] - - def test_set_fields_cycle_fails(self): - class A(Structure): - pass - with pytest.raises(AttributeError): - A._fields_ = [("a", A)] +def test_set_fields_cycle_fails(): + class A(Structure): + pass + with pytest.raises(AttributeError): + A._fields_ = [("a", A)] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py rename to extra_tests/ctypes_tests/test_buffers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py +++ b/extra_tests/ctypes_tests/test_buffers.py @@ -1,71 +1,33 @@ from ctypes import * -from .support import BaseCTypesTestChecker -class TestStringBuffer(BaseCTypesTestChecker): +def test_buffer(): + b = create_string_buffer(32) + assert len(b) == 32 + assert sizeof(b) == 32 * sizeof(c_char) + assert type(b[0]) is bytes - def test_buffer(self): - b = create_string_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(b"abc") + assert len(b) == 4 # trailing nul char + assert sizeof(b) == 4 * sizeof(c_char) + assert type(b[0]) is bytes + assert b[0] == b"a" + assert b[:] == b"abc\0" - b = create_string_buffer(b"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is bytes - assert b[0] == b"a" - assert b[:] == b"abc\0" +def test_from_buffer(): + b1 = bytearray(b"abcde") + b = (c_char * 5).from_buffer(b1) + assert b[2] == b"c" + # + b1 = bytearray(b"abcd") + b = c_int.from_buffer(b1) + assert b.value in (1684234849, # little endian + 1633837924) # big endian - def test_string_conversion(self): - b = create_string_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" - - def test_from_buffer(self): - b1 = bytearray("abcde") - b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" - # - b1 = bytearray("abcd") - b = c_int.from_buffer(b1) - assert b.value in (1684234849, # little endian - 1633837924) # big endian - - def test_from_buffer_keepalive(self): - # Issue #2878 - b1 = bytearray("ab") - array = (c_uint16 * 32)() - array[6] = c_uint16.from_buffer(b1) - # this is also what we get on CPython. I don't think it makes - # sense because the array contains just a copy of the number. - assert array._objects == {'6': b1} - - try: - c_wchar - except NameError: - pass - else: - def test_unicode_buffer(self): - b = create_unicode_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_wchar) - assert type(b[0]) is unicode - - b = create_unicode_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - - def test_unicode_conversion(self): - b = create_unicode_buffer(b"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - +def test_from_buffer_keepalive(): + # Issue #2878 + b1 = bytearray(b"ab") + array = (c_uint16 * 32)() + array[6] = c_uint16.from_buffer(b1) + # this is also what we get on CPython. I don't think it makes + # sense because the array contains just a copy of the number. + assert array._objects == {'6': b1} diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/extra_tests/ctypes_tests/test_callback_traceback.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py rename to extra_tests/ctypes_tests/test_callback_traceback.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py +++ b/extra_tests/ctypes_tests/test_callback_traceback.py @@ -1,80 +1,35 @@ # derived from test_random_things.py -import py +import pytest + from ctypes import * -import sys -def callback_func(arg): - 42 / arg - raise ValueError(arg) +_rawffi = pytest.importorskip('_rawffi') -class TestCallbackTraceback: - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. +# +# This test makes sure the exception types *and* the exception +# value is printed correctly. + + at pytest.mark.skipif("sys.flags.inspect") +def test_SystemExit(monkeypatch, capsys): + """ + When an exception is raised in a ctypes callback function, the C + code prints a traceback. When SystemExit is raised, the interpreter + normally exits immediately. + """ + def callback_func(arg): + raise SystemExit(42) + def custom_exit(value): + raise Exception("<<>>" % (value,)) + monkeypatch.setattr(_rawffi, 'exit', custom_exit) + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) + out, err = capsys.readouterr() + assert not err + cb2(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead a additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import StringIO - old_stderr = sys.stderr - logger = sys.stderr = StringIO.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() - - def test_ValueError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - assert out.splitlines()[-1] == ( - "ValueError: 42") - - def test_IntegerDivisionError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_FloatDivisionError(self): - cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_TypeErrorDivisionError(self): - cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, "spam") - assert out.splitlines()[-1].startswith( - "TypeError: " - "unsupported operand type(s) for") - - def test_SystemExit(self): - import _rawffi - if sys.flags.inspect: - skip("requires sys.flags.inspect == 0") - def callback_func(arg): - raise SystemExit(42) - def custom_exit(value): - raise Exception("<<>>" % (value,)) - original_exit = _rawffi.exit - try: - _rawffi.exit = custom_exit - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) - out = self.capture_stderr(cb2, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - finally: - _rawffi.exit = original_exit + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py rename to extra_tests/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/extra_tests/ctypes_tests/test_callbacks.py @@ -1,283 +1,199 @@ +import pytest + +import math from ctypes import * -import pytest from .support import BaseCTypesTestChecker -class TestCallbacks(BaseCTypesTestChecker): - functype = CFUNCTYPE - -## def tearDown(self): -## import gc -## gc.collect() - - def callback(self, *args): - self.got_args = args - return args[-1] - - def check_type(self, typ, arg): - unwrapped_types = { - c_float: (float,), - c_double: (float,), - c_char: (str,), - c_char_p: (str,), - c_uint: (int, long), - c_ulong: (int, long), - } - - PROTO = self.functype.im_func(typ, typ) - cfunc = PROTO(self.callback) - result = cfunc(arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (arg,) - assert result == arg - - result2 = cfunc(typ(arg)) - assert type(result2) in unwrapped_types.get(typ, (int, long)) - - PROTO = self.functype.im_func(typ, c_byte, typ) - result = PROTO(self.callback)(-3, arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (-3, arg) - assert result == arg - - ################ - - def test_byte(self): - self.check_type(c_byte, 42) - self.check_type(c_byte, -42) - - def test_ubyte(self): - self.check_type(c_ubyte, 42) - - def test_short(self): - self.check_type(c_short, 42) - self.check_type(c_short, -42) - - def test_ushort(self): - self.check_type(c_ushort, 42) - - def test_int(self): - self.check_type(c_int, 42) - self.check_type(c_int, -42) - - def test_uint(self): - self.check_type(c_uint, 42) - - def test_long(self): - self.check_type(c_long, 42) - self.check_type(c_long, -42) - - def test_ulong(self): - self.check_type(c_ulong, 42) - - def test_longlong(self): - self.check_type(c_longlong, 42) - self.check_type(c_longlong, -42) - - def test_ulonglong(self): - self.check_type(c_ulonglong, 42) - - def test_float(self): - # only almost equal: double -> float -> double - import math - self.check_type(c_float, math.e) - self.check_type(c_float, -math.e) - - def test_double(self): - self.check_type(c_double, 3.14) - self.check_type(c_double, -3.14) - - def test_char(self): - self.check_type(c_char, "x") - self.check_type(c_char, "a") - - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. -## def test_char_p(self): -## self.check_type(c_char_p, "abc") -## self.check_type(c_char_p, "def") - - - @pytest.mark.xfail( - reason="we are less strict about callback return type sanity") - def test_unsupported_restype_1(self): - # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. - # POINTER(c_double), for example, is not supported. - - prototype = self.functype.im_func(POINTER(c_double)) - # The type is checked when the prototype is called - with pytest.raises(TypeError): - prototype(lambda: None) - +functypes = [CFUNCTYPE] try: - WINFUNCTYPE + functypes.append(WINFUNCTYPE) except NameError: pass -else: - class TestStdcallCallbacks(TestCallbacks): - functype = WINFUNCTYPE -################################################################ -class TestSampleCallbacks(BaseCTypesTestChecker): +def callback(*args): + callback.got_args = args + return args[-1] - def test_integrate(self): - # Derived from some then non-working code, posted by David Foster - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) +unwrapped_types = { + c_float: (float,), + c_double: (float,), + c_char: (bytes,), + c_char_p: (bytes,), + c_uint: (int,), + c_ulong: (int,), + } - # The function prototype called by 'integrate': double func(double); - CALLBACK = CFUNCTYPE(c_double, c_double) + at pytest.mark.parametrize("typ, arg", [ + (c_byte, 42), + (c_byte, -42), + (c_ubyte, 42), + (c_short, 42), + (c_short, -42), + (c_ushort, 42), + (c_int, 42), + (c_int, -42), + (c_uint, 42), + (c_long, 42), + (c_long, -42), + (c_ulong, 42), + (c_longlong, 42), + (c_longlong, -42), + (c_ulonglong, 42), + (c_float, math.e), # only almost equal: double -> float -> double + (c_float, -math.e), + (c_double, 3.14), + (c_double, -3.14), + (c_char, b"x"), + (c_char, b"a"), +]) + at pytest.mark.parametrize('functype', functypes) +def test_types(typ, arg, functype): + PROTO = functype(typ, typ) + cfunc = PROTO(callback) + result = cfunc(arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (arg,) + assert result == arg - # The integrate function itself, exposed from the _ctypes_test dll - integrate = dll.integrate - integrate.argtypes = (c_double, c_double, CALLBACK, c_long) - integrate.restype = c_double + result2 = cfunc(typ(arg)) + assert type(result2) in unwrapped_types.get(typ, (int,)) - def func(x): - print 'calculating x**2 of',x - return x**2 + PROTO = functype(typ, c_byte, typ) + result = PROTO(callback)(-3, arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (-3, arg) + assert result == arg - result = integrate(0.0, 1.0, CALLBACK(func), 10) - diff = abs(result - 1./3.) + at pytest.mark.parametrize('functype', functypes) +def test_unsupported_restype_1(functype): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. - assert diff < 0.01, "%s not less than 0.01" % diff + prototype = functype(POINTER(c_double)) + # The type is checked when the prototype is called + with pytest.raises(TypeError): + prototype(lambda: None) -################################################################ -class TestMoreCallbacks(BaseCTypesTestChecker): +def test_callback_with_struct_argument(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - def test_callback_with_struct_argument(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] + proto = CFUNCTYPE(c_int, RECT) - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - point.left *= -1 - return point.left+point.top+point.right+point.bottom + def callback(point): + point.left *= -1 + return point.left + point.top + point.right + point.bottom - cbp = proto(callback) + cbp = proto(callback) + rect = RECT(-1000, 100, 10, 1) + res = cbp(rect) + assert res == 1111 + assert rect.left == -1000 # must not have been changed! - rect = RECT(-1000,100,10,1) +def test_callback_from_c_with_struct_argument(dll): + class RECT(Structure): + _fields_ = [("left", c_long), ("top", c_long), + ("right", c_long), ("bottom", c_long)] - res = cbp(rect) + proto = CFUNCTYPE(c_int, RECT) - assert res == 1111 - assert rect.left == -1000 # must not have been changed! + def callback(point): + return point.left + point.top + point.right + point.bottom - def test_callback_from_c_with_struct_argument(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + cbp = proto(callback) + rect = RECT(1000, 100, 10, 1) - class RECT(Structure): - _fields_ = [("left", c_long), ("top", c_long), - ("right", c_long), ("bottom", c_long)] + call_callback_with_rect = dll.call_callback_with_rect + call_callback_with_rect.restype = c_int + call_callback_with_rect.argtypes = [proto, RECT] + res = call_callback_with_rect(cbp, rect) + assert res == 1111 - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - return point.left+point.top+point.right+point.bottom +def test_callback_unsupported_return_struct(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - cbp = proto(callback) - rect = RECT(1000,100,10,1) + proto = CFUNCTYPE(RECT, c_int) + with pytest.raises(TypeError): + proto(lambda r: 0) - call_callback_with_rect = dll.call_callback_with_rect - call_callback_with_rect.restype = c_int - call_callback_with_rect.argtypes = [proto, RECT] - res = call_callback_with_rect(cbp, rect) - assert res == 1111 - def test_callback_unsupported_return_struct(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] +def test_qsort(dll): + PI = POINTER(c_int) + A = c_int*5 + a = A() + for i in range(5): + a[i] = 5-i - proto = CFUNCTYPE(RECT, c_int) - with pytest.raises(TypeError): - proto(lambda r: 0) + assert a[0] == 5 # sanity + def comp(a, b): + a = a.contents.value + b = b.contents.value + if a < b: + return -1 + elif a > b: + return 1 + else: + return 0 + qs = dll.my_qsort + qs.restype = None + CMP = CFUNCTYPE(c_int, PI, PI) + qs.argtypes = (PI, c_size_t, c_size_t, CMP) - def test_qsort(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) - PI = POINTER(c_int) - A = c_int*5 - a = A() - for i in range(5): - a[i] = 5-i + res = list(a) - assert a[0] == 5 # sanity + assert res == [1,2,3,4,5] - def comp(a, b): - a = a.contents.value - b = b.contents.value - return cmp(a,b) - qs = dll.my_qsort - qs.restype = None - CMP = CFUNCTYPE(c_int, PI, PI) - qs.argtypes = (PI, c_size_t, c_size_t, CMP) +def test_pyobject_as_opaque(dll): + def callback(arg): + return arg() - qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) + CTP = CFUNCTYPE(c_int, py_object) + cfunc = dll._testfunc_callback_opaque + cfunc.argtypes = [CTP, py_object] + cfunc.restype = c_int + res = cfunc(CTP(callback), lambda : 3) + assert res == 3 - res = list(a) +def test_callback_void(capsys, dll): + def callback(): + pass - assert res == [1,2,3,4,5] + CTP = CFUNCTYPE(None) + cfunc = dll._testfunc_callback_void + cfunc.argtypes = [CTP] + cfunc.restype = int + cfunc(CTP(callback)) + out, err = capsys.readouterr() + assert (out, err) == ("", "") - def test_pyobject_as_opaque(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - def callback(arg): - return arg() +def test_callback_pyobject(): + def callback(obj): + return obj - CTP = CFUNCTYPE(c_int, py_object) - cfunc = dll._testfunc_callback_opaque - cfunc.argtypes = [CTP, py_object] - cfunc.restype = c_int - res = cfunc(CTP(callback), lambda : 3) - assert res == 3 + FUNC = CFUNCTYPE(py_object, py_object) + cfunc = FUNC(callback) + param = c_int(42) + assert cfunc(param) is param - def test_callback_void(self, capsys): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - - def callback(): - pass - - CTP = CFUNCTYPE(None) - cfunc = dll._testfunc_callback_void - cfunc.argtypes = [CTP] - cfunc.restype = int - cfunc(CTP(callback)) - out, err = capsys.readouterr() - assert (out, err) == ("", "") - - - def test_callback_pyobject(self): - def callback(obj): - return obj - - FUNC = CFUNCTYPE(py_object, py_object) - cfunc = FUNC(callback) - param = c_int(42) - assert cfunc(param) is param - - def test_raise_argumenterror(self): - def callback(x): - pass - FUNC = CFUNCTYPE(None, c_void_p) - cfunc = FUNC(callback) - param = c_uint(42) - with pytest.raises(ArgumentError): - cfunc(param) +def test_raise_argumenterror(): + def callback(x): + pass + FUNC = CFUNCTYPE(None, c_void_p) + cfunc = FUNC(callback) + param = c_uint(42) + with pytest.raises(ArgumentError): + cfunc(param) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/extra_tests/ctypes_tests/test_cast.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_cast.py rename to extra_tests/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/extra_tests/ctypes_tests/test_cast.py @@ -1,106 +1,30 @@ +import pytest + from ctypes import * -import sys, py -from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - mod.lib = CDLL(str(conftest.sofile)) +def test_cast_functype(dll): + # make sure we can cast function type + my_sqrt = dll.my_sqrt + saved_objects = my_sqrt._objects.copy() + sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) + assert sqrt(4.0) == 2.0 + assert not cast(0, CFUNCTYPE(c_int)) + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) -class TestCast(BaseCTypesTestChecker): +def test_cast_argumenterror(): + param = c_uint(42) + with pytest.raises(ArgumentError): + cast(param, c_void_p) - def test_array2pointer(self): - array = (c_int * 3)(42, 17, 2) - - # casting an array to a pointer works. - ptr = cast(array, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - if 2*sizeof(c_short) == sizeof(c_int): - ptr = cast(array, POINTER(c_short)) - if sys.byteorder == "little": - assert [ptr[i] for i in range(6)] == ( - [42, 0, 17, 0, 2, 0]) - else: - assert [ptr[i] for i in range(6)] == ( - [0, 42, 0, 17, 0, 2]) - - def test_address2pointer(self): - array = (c_int * 3)(42, 17, 2) - - address = addressof(array) - ptr = cast(c_void_p(address), POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - ptr = cast(address, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - def test_p2a_objects(self): - py.test.skip("we make copies of strings") - array = (c_char_p * 5)() - assert array._objects is None - array[0] = "foo bar" - assert array._objects == {'0': "foo bar"} - - p = cast(array, POINTER(c_char_p)) - # array and p share a common _objects attribute - assert p._objects is array._objects - assert array._objects == {'0': "foo bar", id(array): array} - p[0] = "spam spam" - assert p._objects == {'0': "spam spam", id(array): array} - assert array._objects is p._objects - p[1] = "foo bar" - assert p._objects == {'1': 'foo bar', '0': "spam spam", id(array): array} - assert array._objects is p._objects - - def test_other(self): - p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) - assert p[:4] == [1,2, 3, 4] - c_int() - assert p[:4] == [1, 2, 3, 4] - p[2] = 96 - assert p[:4] == [1, 2, 96, 4] - c_int() - assert p[:4] == [1, 2, 96, 4] - - def test_char_p(self): - # This didn't work: bad argument to internal function - s = c_char_p("hiho") - - assert cast(cast(s, c_void_p), c_char_p).value == ( - "hiho") - - try: - c_wchar_p - except NameError: - pass - else: - def test_wchar_p(self): - s = c_wchar_p("hiho") - assert cast(cast(s, c_void_p), c_wchar_p).value == ( - "hiho") - - def test_cast_functype(self): - # make sure we can cast function type - my_sqrt = lib.my_sqrt - saved_objects = my_sqrt._objects.copy() - sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) - assert sqrt(4.0) == 2.0 - assert not cast(0, CFUNCTYPE(c_int)) - # - assert sqrt._objects is my_sqrt._objects # on CPython too - my_sqrt._objects.clear() - my_sqrt._objects.update(saved_objects) - - def test_cast_argumenterror(self): - param = c_uint(42) - py.test.raises(ArgumentError, "cast(param, c_void_p)") - - def test_c_bool(self): - x = c_bool(42) - assert x.value is True - x = c_bool(0.0) - assert x.value is False - x = c_bool("") - assert x.value is False - x = c_bool(['yadda']) - assert x.value is True +def test_c_bool(): + x = c_bool(42) + assert x.value is True + x = c_bool(0.0) + assert x.value is False + x = c_bool("") + assert x.value is False + x = c_bool(['yadda']) + assert x.value is True diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py b/extra_tests/ctypes_tests/test_commethods.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py rename to extra_tests/ctypes_tests/test_commethods.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/extra_tests/ctypes_tests/test_errno.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_errno.py rename to extra_tests/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/extra_tests/ctypes_tests/test_errno.py @@ -1,26 +1,19 @@ -import py +import pytest import ctypes -from _ctypes import function +_rawffi = pytest.importorskip('_rawffi') # PyPy-only -try: - import _rawffi -except ImportError: - py.test.skip("app-level test only for PyPy") +def test_errno_saved_and_restored(): + def check(): + assert _rawffi.get_errno() == 42 + assert ctypes.get_errno() == old + check.free_temp_buffers = lambda *args: None + f = ctypes._CFuncPtr() + old = _rawffi.get_errno() + f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO + ctypes.set_errno(42) + f._call_funcptr(check) + assert _rawffi.get_errno() == old + ctypes.set_errno(0) -class TestErrno: - - def test_errno_saved_and_restored(self): - def check(): - assert _rawffi.get_errno() == 42 - assert ctypes.get_errno() == old - check.free_temp_buffers = lambda *args: None - f = function.CFuncPtr() - old = _rawffi.get_errno() - f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO - ctypes.set_errno(42) - f._call_funcptr(check) - assert _rawffi.get_errno() == old - ctypes.set_errno(0) - - # see also test_functions.test_errno +# see also test_functions.test_errno diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_extra.py rename to extra_tests/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/extra_tests/ctypes_tests/test_extra.py @@ -5,244 +5,239 @@ import py from ctypes import * -from .support import BaseCTypesTestChecker -class TestExtra(BaseCTypesTestChecker): - def test_primitive_pointer(self): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 +def test_primitive_pointer(): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 - ## *** the rest is commented out because it should work but occasionally - ## *** trigger a ctypes bug (SourceForge bug #1467852). *** - ## q = pointer(y) - ## pp[0] = q # pp ---> p ---> y = 14 - ## assert y.value == 14 # (^^^ not q! ) - ## assert p.contents.value == 14 - ## assert pp.contents.contents.value == 14 - ## q.contents = x - ## assert pp.contents.contents.value == 14 +## *** the rest is commented out because it should work but occasionally +## *** trigger a ctypes bug (SourceForge bug #1467852). *** +## q = pointer(y) +## pp[0] = q # pp ---> p ---> y = 14 +## assert y.value == 14 # (^^^ not q! ) +## assert p.contents.value == 14 +## assert pp.contents.contents.value == 14 +## q.contents = x +## assert pp.contents.contents.value == 14 - def test_char_p(self): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" +def test_char_p(): + x = c_char_p(b"hello\x00world") + assert x.value == b"hello" + x.value = b"world" + assert x.value == b"world" - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + p = pointer(x) + assert p[0] == x.value == b"world" + p[0] = b"other" + assert x.value == p.contents.value == p[0] == b"other" - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + myarray = (c_char_p * 10)() + myarray[7] = b"hello" + assert isinstance(myarray[7], bytes) + assert myarray[7] == b"hello" - def test_struct(self): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] +def test_struct(): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 - def test_ptr_array(self): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) +def test_ptr_array(): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 - def test_void_p(self): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 +def test_void_p(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 - def test_char_array(self): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' +def test_char_array(): + a = (c_char * 3)() + a[0] = b'x' + a[1] = b'y' + assert a.value == b'xy' + a[2] = b'z' + assert a.value == b'xyz' - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = b"nxw" + assert b[0] == b'n' + assert b[1] == b'x' + assert b[2] == b'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = b"?" + assert b[0] == b'?' + assert b[1] == b'\x00' + assert b[2] == b'w' - class S(Structure): - _fields_ = [('p', POINTER(c_char))] + class S(Structure): + _fields_ = [('p', POINTER(c_char))] - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s = S() + s.p = b + s.p.contents.value = b'!' + assert b.value == b'!' - assert len(create_string_buffer(0)) == 0 + assert len(create_string_buffer(0)) == 0 - def test_array(self): - a = (c_int * 10)() +def test_array(): + a = (c_int * 10)() - class S(Structure): - _fields_ = [('p', POINTER(c_int))] + class S(Structure): + _fields_ = [('p', POINTER(c_int))] - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] - def test_truth_value(self): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... +def test_truth_value(): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) - assert not c_double(0.0) - assert not c_ulonglong(0) - assert c_ulonglong(2**42) + assert c_int(12) From pypy.commits at gmail.com Sun Dec 16 02:48:57 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 15 Dec 2018 23:48:57 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #394 Message-ID: <5c160369.1c69fb81.5ba36.b8aa@mx.google.com> Author: Armin Rigo Branch: Changeset: r3175:f4de3d6b3c34 Date: 2018-12-16 08:48 +0100 http://bitbucket.org/cffi/cffi/changeset/f4de3d6b3c34/ Log: Issue #394 Implement ffi.from_buffer(x, require_writable=True) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -6741,7 +6741,8 @@ return 0; } -static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x) +static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x, + int require_writable) { CDataObject *cd; Py_buffer *view; @@ -6761,7 +6762,7 @@ PyErr_NoMemory(); return NULL; } - if (_my_PyObject_GetContiguousBuffer(x, view, 0) < 0) + if (_my_PyObject_GetContiguousBuffer(x, view, require_writable) < 0) goto error1; cd = (CDataObject *)PyObject_GC_New(CDataObject_owngc_frombuf, @@ -6789,15 +6790,17 @@ { CTypeDescrObject *ct; PyObject *x; - - if (!PyArg_ParseTuple(args, "O!O", &CTypeDescr_Type, &ct, &x)) + int require_writable = 0; + + if (!PyArg_ParseTuple(args, "O!O|i", &CTypeDescr_Type, &ct, &x, + &require_writable)) return NULL; if (!(ct->ct_flags & CT_IS_UNSIZED_CHAR_A)) { PyErr_Format(PyExc_TypeError, "needs 'char[]', got '%s'", ct->ct_name); return NULL; } - return direct_from_buffer(ct, x); + return direct_from_buffer(ct, x, require_writable); } static int _fetch_as_buffer(PyObject *x, Py_buffer *view, int writable_only) diff --git a/c/ffi_obj.c b/c/ffi_obj.c --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -697,9 +697,16 @@ "containing large quantities of raw data in some other format, like\n" "'array.array' or numpy arrays."); -static PyObject *ffi_from_buffer(PyObject *self, PyObject *arg) +static PyObject *ffi_from_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - return direct_from_buffer(g_ct_chararray, arg); + PyObject *arg; + int require_writable = 0; + static char *keywords[] = {"python_buffer", "require_writable", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:from_buffer", keywords, + &arg, &require_writable)) + return NULL; + return direct_from_buffer(g_ct_chararray, arg, require_writable); } PyDoc_STRVAR(ffi_gc_doc, @@ -1072,7 +1079,7 @@ {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, {"dlclose", (PyCFunction)ffi_dlclose, METH_VARARGS, ffi_dlclose_doc}, {"dlopen", (PyCFunction)ffi_dlopen, METH_VARARGS, ffi_dlopen_doc}, - {"from_buffer",(PyCFunction)ffi_from_buffer,METH_O, ffi_from_buffer_doc}, + {"from_buffer",(PyCFunction)ffi_from_buffer,METH_VKW, ffi_from_buffer_doc}, {"from_handle",(PyCFunction)ffi_from_handle,METH_O, ffi_from_handle_doc}, {"gc", (PyCFunction)ffi_gc, METH_VKW, ffi_gc_doc}, {"getctype", (PyCFunction)ffi_getctype, METH_VKW, ffi_getctype_doc}, diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3741,6 +3741,18 @@ check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") +def test_from_buffer_require_writable(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p1 = from_buffer(BCharA, b"foo", False) + assert p1 == from_buffer(BCharA, b"foo", False) + py.test.raises((TypeError, BufferError), from_buffer, BCharA, b"foo", True) + ba = bytearray(b"foo") + p1 = from_buffer(BCharA, ba, True) + p1[0] = b"g" + assert ba == b"goo" + def test_memmove(): Short = new_primitive_type("short") ShortA = new_array_type(new_pointer_type(Short), None) diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -341,7 +341,7 @@ # """ # note that 'buffer' is a type, set on this instance by __init__ - def from_buffer(self, python_buffer): + def from_buffer(self, python_buffer, require_writable=False): """Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is not meant to be used on the built-in types @@ -349,7 +349,8 @@ but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ - return self._backend.from_buffer(self.BCharA, python_buffer) + return self._backend.from_buffer(self.BCharA, python_buffer, + require_writable) def memmove(self, dest, src, n): """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -188,7 +188,8 @@ *New in version 1.10:* ``ffi.buffer`` is now the type of the returned buffer objects; ``ffi.buffer()`` actually calls the constructor. -**ffi.from_buffer(python_buffer)**: return a ```` that +**ffi.from_buffer(python_buffer, require_writable=False)**: +return a ```` that points to the data of the given Python object, which must support the buffer interface. This is the opposite of ``ffi.buffer()``. It gives a reference to the existing data, not a copy. @@ -216,6 +217,18 @@ resize the bytearray, the ```` object will point to freed memory); and byte strings were supported in version 1.8 onwards. +*New in version 1.12:* added the ``require_writable`` argument. If set to +True, the function fails if the buffer obtained from ``python_buffer`` is +read-only (e.g. if ``python_buffer`` is a byte string). The exact exception is +raised by the object itself, and for things like bytes it varies with the +Python version, so don't rely on it. (Before version 1.12, the same effect can +be achieved with a hack: call ``ffi.memmove(python_buffer, b"", 0)``. This has +no effect if the object is writable, but fails if it is read-only.) + +Please keep in mind that CFFI does not implement the C keyword ``const``: even +if you set ``require_writable`` to False explicitly, you still get a regular +read-write cdata pointer. + ffi.memmove() +++++++++++++ diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -23,6 +23,10 @@ * CPython 2.x: ``ffi.dlopen()`` failed with non-ascii file names on Posix +* ``ffi.from_buffer()`` takes a new keyword argument ``require_writable``. + When set to True, it asks the object passed in to raise an exception if + it is read-only. + v1.11.5 ======= diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py --- a/testing/cffi0/test_ffi_backend.py +++ b/testing/cffi0/test_ffi_backend.py @@ -326,6 +326,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(self): ffi = FFI() diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py --- a/testing/cffi1/test_ffi_obj.py +++ b/testing/cffi1/test_ffi_obj.py @@ -243,6 +243,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(): ffi = _cffi1_backend.FFI() diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py --- a/testing/cffi1/test_new_ffi_1.py +++ b/testing/cffi1/test_new_ffi_1.py @@ -1653,6 +1653,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_all_primitives(self): assert set(PRIMITIVE_TO_INDEX) == set([ From pypy.commits at gmail.com Sun Dec 16 02:53:09 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 15 Dec 2018 23:53:09 -0800 (PST) Subject: [pypy-commit] pypy default: Update to cffi/f4de3d6b3c34: implement ffi.from_buffer(x, require_writable=True) Message-ID: <5c160465.1c69fb81.dba3a.6021@mx.google.com> Author: Armin Rigo Branch: Changeset: r95502:70d98908b3fe Date: 2018-12-16 08:52 +0100 http://bitbucket.org/pypy/pypy/changeset/70d98908b3fe/ Log: Update to cffi/f4de3d6b3c34: implement ffi.from_buffer(x, require_writable=True) diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -327,6 +327,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(self): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py --- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py +++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py @@ -244,6 +244,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(): ffi = _cffi1_backend.FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -1654,6 +1654,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_all_primitives(self): assert set(PRIMITIVE_TO_INDEX) == set([ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -341,7 +341,7 @@ # """ # note that 'buffer' is a type, set on this instance by __init__ - def from_buffer(self, python_buffer): + def from_buffer(self, python_buffer, require_writable=False): """Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is not meant to be used on the built-in types @@ -349,7 +349,8 @@ but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ - return self._backend.from_buffer(self.BCharA, python_buffer) + return self._backend.from_buffer(self.BCharA, python_buffer, + require_writable) def memmove(self, dest, src, n): """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -327,7 +327,8 @@ return w_ctype.cast(w_ob) - def descr_from_buffer(self, w_python_buffer): + @unwrap_spec(require_writable=int) + def descr_from_buffer(self, w_python_buffer, require_writable=0): """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is @@ -337,7 +338,8 @@ 'array.array' or numpy arrays.""" # w_ctchara = newtype._new_chara_type(self.space) - return func._from_buffer(self.space, w_ctchara, w_python_buffer) + return func._from_buffer(self.space, w_ctchara, w_python_buffer, + require_writable) @unwrap_spec(w_arg=W_CData) diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -110,8 +110,8 @@ def _fetch_as_write_buffer(space, w_x): return space.writebuf_w(w_x) - at unwrap_spec(w_ctype=ctypeobj.W_CType) -def from_buffer(space, w_ctype, w_x): + at unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int) +def from_buffer(space, w_ctype, w_x, require_writable=0): from pypy.module._cffi_backend import ctypearray, ctypeprim # if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or @@ -119,13 +119,16 @@ raise oefmt(space.w_TypeError, "needs 'char[]', got '%s'", w_ctype.name) # - return _from_buffer(space, w_ctype, w_x) + return _from_buffer(space, w_ctype, w_x, require_writable) -def _from_buffer(space, w_ctype, w_x): +def _from_buffer(space, w_ctype, w_x, require_writable): if space.isinstance_w(w_x, space.w_unicode): raise oefmt(space.w_TypeError, - "from_buffer() cannot return the address a unicode") - buf = _fetch_as_read_buffer(space, w_x) + "from_buffer() cannot return the address of a unicode object") + if require_writable: + buf = _fetch_as_write_buffer(space, w_x) + else: + buf = _fetch_as_read_buffer(space, w_x) if space.isinstance_w(w_x, space.w_bytes): _cdata = get_raw_address_of_string(space, w_x) else: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3730,6 +3730,18 @@ check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") +def test_from_buffer_require_writable(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p1 = from_buffer(BCharA, b"foo", False) + assert p1 == from_buffer(BCharA, b"foo", False) + py.test.raises((TypeError, BufferError), from_buffer, BCharA, b"foo", True) + ba = bytearray(b"foo") + p1 = from_buffer(BCharA, ba, True) + p1[0] = b"g" + assert ba == b"goo" + def test_memmove(): Short = new_primitive_type("short") ShortA = new_array_type(new_pointer_type(Short), None) diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -287,6 +287,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(self): import sys From pypy.commits at gmail.com Sun Dec 16 11:03:13 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 16 Dec 2018 08:03:13 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 Message-ID: <5c167741.1c69fb81.a7166.f675@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.6 Changeset: r95503:df88cbf79dc7 Date: 2018-12-16 17:02 +0100 http://bitbucket.org/pypy/pypy/changeset/df88cbf79dc7/ Log: merge py3.5 diff too long, truncating to 2000 out of 9468 lines diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -5,7 +5,7 @@ from cffi import recompiler from extra_tests.cffi_tests.udir import udir from extra_tests.cffi_tests.support import u, long -from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture, _verify try: import importlib @@ -36,7 +36,7 @@ # add '-Werror' to the existing 'extra_compile_args' flags kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + ['-Werror']) - return recompiler._verify(ffi, module_name, source, *args, **kwds) + return _verify(ffi, module_name, source, *args, **kwds) def test_set_source_no_slashes(): ffi = FFI() @@ -1539,15 +1539,18 @@ assert (pt.x, pt.y) == (99*500*999, -99*500*999) def test_extern_python_1(): + import warnings ffi = FFI() - ffi.cdef(""" + with warnings.catch_warnings(record=True) as log: + ffi.cdef(""" extern "Python" { int bar(int, int); void baz(int, int); int bok(void); void boz(void); } - """) + """) + assert len(log) == 0, "got a warning: %r" % (log,) lib = verify(ffi, 'test_extern_python_1', """ static void baz(int, int); /* forward */ """) diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py --- a/extra_tests/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -4,6 +4,7 @@ from cffi import CDefError from cffi import recompiler from extra_tests.cffi_tests.support import * +from extra_tests.cffi_tests.support import _verify import _cffi_backend lib_m = ['m'] @@ -38,9 +39,8 @@ except AttributeError: pass self.set_source(module_name, preamble) - return recompiler._verify(self, module_name, preamble, *args, - extra_compile_args=self._extra_compile_args, - **kwds) + return _verify(self, module_name, preamble, *args, + extra_compile_args=self._extra_compile_args, **kwds) class FFI_warnings_not_error(FFI): _extra_compile_args = [] diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py --- a/extra_tests/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -62,3 +62,28 @@ def getvalue(self): return self._value + +def _verify(ffi, module_name, preamble, *args, **kwds): + import imp + from cffi.recompiler import recompile + from .udir import udir + assert module_name not in sys.modules, "module name conflict: %r" % ( + module_name,) + kwds.setdefault('tmpdir', str(udir)) + outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) + module = imp.load_dynamic(module_name, outputfilename) + # + # hack hack hack: copy all *bound methods* from module.ffi back to the + # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). + for name in dir(module.ffi): + if not name.startswith('_'): + attr = getattr(module.ffi, name) + if attr is not getattr(ffi, name, object()): + setattr(ffi, name, attr) + def typeof_disabled(*args, **kwds): + raise NotImplementedError + ffi._typeof = typeof_disabled + for name in dir(ffi): + if not name.startswith('_') and not hasattr(module.ffi, name): + setattr(ffi, name, NotImplemented) + return module.lib diff --git a/pypy/module/test_lib_pypy/ctypes_tests/__init__.py b/extra_tests/ctypes_tests/__init__.py rename from pypy/module/test_lib_pypy/ctypes_tests/__init__.py rename to extra_tests/ctypes_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/extra_tests/ctypes_tests/_ctypes_test.c rename from pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c rename to extra_tests/ctypes_tests/_ctypes_test.c diff --git a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py b/extra_tests/ctypes_tests/conftest.py rename from pypy/module/test_lib_pypy/ctypes_tests/conftest.py rename to extra_tests/ctypes_tests/conftest.py --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ b/extra_tests/ctypes_tests/conftest.py @@ -3,10 +3,6 @@ import sys import os -def pytest_ignore_collect(path): - if '__pypy__' not in sys.builtin_module_names: - return True - # XXX: copied from pypy/tool/cpyext/extbuild.py if os.name != 'nt': so_ext = 'so' @@ -85,8 +81,7 @@ return outputfilename # end copy -def compile_so_file(): - udir = pytest.ensuretemp('_ctypes_test') +def compile_so_file(udir): cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': @@ -96,8 +91,12 @@ return c_compile([cfile], str(udir / '_ctypes_test'), libraries=libraries) -# we need to run after the "tmpdir" plugin which installs pytest.ensuretemp - at pytest.mark.trylast -def pytest_configure(config): - global sofile - sofile = compile_so_file() + at pytest.fixture(scope='session') +def sofile(tmpdir_factory): + udir = tmpdir_factory.mktemp('_ctypes_test') + return str(compile_so_file(udir)) + + at pytest.fixture +def dll(sofile): + from ctypes import CDLL + return CDLL(str(sofile)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/extra_tests/ctypes_tests/support.py rename from pypy/module/test_lib_pypy/ctypes_tests/support.py rename to extra_tests/ctypes_tests/support.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/extra_tests/ctypes_tests/test_anon.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_anon.py rename to extra_tests/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/extra_tests/ctypes_tests/test_anon.py @@ -1,86 +1,55 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -class TestAnon(BaseCTypesTestChecker): + at pytest.mark.pypy_only +def test_nested(): + class ANON_S(Structure): + _fields_ = [("a", c_int)] - def test_anon(self): - class ANON(Union): - _fields_ = [("a", c_int), - ("b", c_int)] + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON), - ("y", c_int)] - _anonymous_ = ["_"] + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) + assert Y.x.offset == 0 + assert Y.a.offset == sizeof(c_int) + assert Y.b.offset == sizeof(c_int) + assert Y._.offset == sizeof(c_int) + assert Y.y.offset == sizeof(c_int) * 2 - assert ANON.a.offset == 0 - assert ANON.b.offset == 0 + assert Y._names_ == ['x', 'a', 'b', 'y'] - def test_anon_nonseq(self): - # TypeError: _anonymous_ must be a sequence - with pytest.raises(TypeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": 42}) +def test_anonymous_fields_on_instance(): + # this is about the *instance-level* access of anonymous fields, + # which you'd guess is the most common, but used not to work + # (issue #2230) - def test_anon_nonmember(self): - # AttributeError: type object 'Name' has no attribute 'x' - with pytest.raises(AttributeError): - type(Structure)( - "Name", (Structure,), {"_fields_": [], "_anonymous_": ["x"]}) + class B(Structure): + _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] + class A(Structure): + _anonymous_ = ["b"] + _fields_ = [("b", B)] - def test_nested(self): - class ANON_S(Structure): - _fields_ = [("a", c_int)] + a = A() + a.x = 5 + assert a.x == 5 + assert a.b.x == 5 + a.b.x += 1 + assert a.x == 6 - class ANON_U(Union): - _fields_ = [("_", ANON_S), - ("b", c_int)] - _anonymous_ = ["_"] + class C(Structure): + _anonymous_ = ["a"] + _fields_ = [("v", c_int), ("a", A)] - class Y(Structure): - _fields_ = [("x", c_int), - ("_", ANON_U), - ("y", c_int)] - _anonymous_ = ["_"] - - assert Y.x.offset == 0 - assert Y.a.offset == sizeof(c_int) - assert Y.b.offset == sizeof(c_int) - assert Y._.offset == sizeof(c_int) - assert Y.y.offset == sizeof(c_int) * 2 - - assert Y._names_ == ['x', 'a', 'b', 'y'] - - def test_anonymous_fields_on_instance(self): - # this is about the *instance-level* access of anonymous fields, - # which you'd guess is the most common, but used not to work - # (issue #2230) - - class B(Structure): - _fields_ = [("x", c_int), ("y", c_int), ("z", c_int)] - class A(Structure): - _anonymous_ = ["b"] - _fields_ = [("b", B)] - - a = A() - a.x = 5 - assert a.x == 5 - assert a.b.x == 5 - a.b.x += 1 - assert a.x == 6 - - class C(Structure): - _anonymous_ = ["a"] - _fields_ = [("v", c_int), ("a", A)] - - c = C() - c.v = 3 - c.y = -8 - assert c.v == 3 - assert c.y == c.a.y == c.a.b.y == -8 - assert not hasattr(c, 'b') + c = C() + c.v = 3 + c.y = -8 + assert c.v == 3 + assert c.y == c.a.y == c.a.b.y == -8 + assert not hasattr(c, 'b') diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/extra_tests/ctypes_tests/test_array.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_array.py rename to extra_tests/ctypes_tests/test_array.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py +++ b/extra_tests/ctypes_tests/test_array.py @@ -1,177 +1,64 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -formats = "bBhHiIlLqQfd" +def test_slice(): + values = list(range(5)) + numarray = c_int * 5 -formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ - c_long, c_ulonglong, c_float, c_double + na = numarray(*(c_int(x) for x in values)) -class TestArray(BaseCTypesTestChecker): - def test_simple(self): - # create classes holding simple numeric types, and check - # various properties. + assert list(na[0:0]) == [] + assert list(na[:]) == values + assert list(na[:10]) == values - init = range(15, 25) +def test_init_again(): + sz = (c_char * 3)() + addr1 = addressof(sz) + sz.__init__(*b"foo") + addr2 = addressof(sz) + assert addr1 == addr2 - for fmt in formats: - alen = len(init) - int_array = ARRAY(fmt, alen) +def test_array_of_structures(): + class X(Structure): + _fields_ = [('x', c_int), ('y', c_int)] - ia = int_array(*init) - # length of instance ok? - assert len(ia) == alen + Y = X * 2 + y = Y() + x = X() + x.y = 3 + y[1] = x + assert y[1].y == 3 - # slot values ok? - values = [ia[i] for i in range(len(init))] - assert values == init +def test_output_simple(): + A = c_char * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # change the items - from operator import setitem - new_values = range(42, 42+alen) - [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(len(init))] - assert values == new_values + A = c_wchar * 10 + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' - # are the items initialized to 0? - ia = int_array() - values = [ia[i] for i in range(len(init))] - assert values == [0] * len(init) +def test_output_simple_array(): + A = c_char * 10 + AA = A * 10 + aa = AA() + assert aa[0] != b'' - # Too many in itializers should be caught - with pytest.raises(IndexError): - int_array(*range(alen*2)) +def test_output_complex_test(): + class Car(Structure): + _fields_ = [("brand", c_char * 10), + ("speed", c_float), + ("owner", c_char * 10)] - CharArray = ARRAY(c_char, 3) + assert isinstance(Car(b"abcdefghi", 42.0, b"12345").brand, bytes) + assert Car(b"abcdefghi", 42.0, b"12345").brand == b"abcdefghi" + assert Car(b"abcdefghio", 42.0, b"12345").brand == b"abcdefghio" + with pytest.raises(ValueError): + Car(b"abcdefghiop", 42.0, b"12345") - ca = CharArray("a", "b", "c") - - # Should this work? It doesn't: - # CharArray("abc") - with pytest.raises(TypeError): - CharArray("abc") - - assert ca[0] == "a" - assert ca[1] == "b" - assert ca[2] == "c" - assert ca[-3] == "a" - assert ca[-2] == "b" - assert ca[-1] == "c" - - assert len(ca) == 3 - - # slicing is now supported, but not extended slicing (3-argument)! - from operator import getslice, delitem - with pytest.raises(TypeError): - getslice(ca, 0, 1, -1) - - # cannot delete items - with pytest.raises(TypeError): - delitem(ca, 0) - - def test_numeric_arrays(self): - - alen = 5 - - numarray = ARRAY(c_int, alen) - - na = numarray() - values = [na[i] for i in range(alen)] - assert values == [0] * alen - - na = numarray(*[c_int()] * alen) - values = [na[i] for i in range(alen)] - assert values == [0]*alen - - na = numarray(1, 2, 3, 4, 5) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - na = numarray(*map(c_int, (1, 2, 3, 4, 5))) - values = [i for i in na] - assert values == [1, 2, 3, 4, 5] - - def test_slice(self): - values = range(5) - numarray = c_int * 5 - - na = numarray(*(c_int(x) for x in values)) - - assert list(na[0:0]) == [] - assert list(na[:]) == values - assert list(na[:10]) == values - - def test_classcache(self): - assert not ARRAY(c_int, 3) is ARRAY(c_int, 4) - assert ARRAY(c_int, 3) is ARRAY(c_int, 3) - - def test_from_address(self): - # Failed with 0.9.8, reported by JUrner - p = create_string_buffer("foo") - sz = (c_char * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - - def test_init_again(self): - sz = (c_char * 3)() - addr1 = addressof(sz) - sz.__init__(*"foo") - addr2 = addressof(sz) - assert addr1 == addr2 - - try: - create_unicode_buffer - except NameError: - pass - else: - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - assert sz[:] == "foo" - assert sz.value == "foo" - -class TestSophisticatedThings(BaseCTypesTestChecker): - def test_array_of_structures(self): - class X(Structure): - _fields_ = [('x', c_int), ('y', c_int)] - - Y = X * 2 - y = Y() - x = X() - x.y = 3 - y[1] = x - assert y[1].y == 3 - - def test_output_simple(self): - A = c_char * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - A = c_wchar * 10 - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' - - def test_output_simple_array(self): - A = c_char * 10 - AA = A * 10 - aa = AA() - assert aa[0] != '' - - def test_output_complex_test(self): - class Car(Structure): - _fields_ = [("brand", c_char * 10), - ("speed", c_float), - ("owner", c_char * 10)] - - assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes) - assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi" - assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio" - with pytest.raises(ValueError): - Car("abcdefghiop", 42.0, "12345") - - A = Car._fields_[2][1] - TP = POINTER(A) - x = TP(A()) - assert x[0] != '' + A = Car._fields_[2][1] + TP = POINTER(A) + x = TP(A()) + assert x[0] != b'' diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py b/extra_tests/ctypes_tests/test_base.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_base.py rename to extra_tests/ctypes_tests/test_base.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_base.py +++ b/extra_tests/ctypes_tests/test_base.py @@ -1,26 +1,24 @@ -from .support import WhiteBoxTests - +import pytest from ctypes import * -# WhiteBoxTests +pytestmark = pytest.mark.pypy_only -class TestCTypesBase(WhiteBoxTests): - def test_pointer(self): - p = pointer(pointer(c_int(2))) - x = p[0] - assert x._base is p +def test_pointer(): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p - def test_structure(self): - class X(Structure): - _fields_ = [('x', POINTER(c_int)), - ('y', POINTER(c_int))] +def test_structure(): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] - x = X() - assert x.y._base is x - assert x.y._index == 1 + x = X() + assert x.y._base is x + assert x.y._index == 1 - def test_array(self): - X = POINTER(c_int) * 24 - x = X() - assert x[16]._base is x - assert x[16]._index == 16 +def test_array(): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py b/extra_tests/ctypes_tests/test_bitfields.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py rename to extra_tests/ctypes_tests/test_bitfields.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_bitfields.py +++ b/extra_tests/ctypes_tests/test_bitfields.py @@ -1,249 +1,19 @@ import pytest from ctypes import * -from .support import BaseCTypesTestChecker -import os -import ctypes -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types +def test_set_fields_attr(): + class A(Structure): + pass + A._fields_ = [("a", c_byte), ("b", c_ubyte)] +def test_set_fields_attr_bitfields(): + class A(Structure): + pass + A._fields_ = [("a", POINTER(A)), ("b", c_ubyte, 4)] -def setup_module(mod): - import conftest - _ctypes_test = str(conftest.sofile) - func = CDLL(_ctypes_test).unpack_bitfields - func.argtypes = POINTER(BITS), c_char - mod.func = func - - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - - -class TestC: - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - def test_shorts(self): - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - assert (name, i, getattr(b, name)) == (name, i, func(byref(b), name)) - - -class TestBitField: - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - x.a, x.b, x.c = -1, 7, -1 - assert (x.a, x.b, x.c) == (-1, 7, -1) - - x = X() - x.a, x.b, x.c = -1, -7, -1 - assert (x.a, x.b, x.c) == (-1, -7, -1) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - assert sizeof(X) == sizeof(c_longlong) - x = X() - assert (x.a, x.b, x.c) == (0, 0, 0) - x.a, x.b, x.c = 7, 2305843009213693953, 7 - assert (x.a, x.b, x.c) == (1, 2305843009213693953, 1) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ)*2 - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, -1, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, -1, 0) - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - x = X() - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 0, 0) - x.a = -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 7, 0, 0) - x.a, x.b = 0, -1 - assert (c_typ, x.a, x.b, x.c) == (c_typ, 0, 7, 0) - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char_p') - - result = self.fail_fields(("a", c_void_p, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_void_p') - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - assert result == (TypeError, 'bit fields not allowed for type LP_c_int') - - result = self.fail_fields(("a", c_char, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_char') - - try: - c_wchar - except NameError: - pass - else: - result = self.fail_fields(("a", c_wchar, 1)) - assert result == (TypeError, 'bit fields not allowed for type c_wchar') - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - assert result == (TypeError, 'bit fields not allowed for type Dummy') - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - result = self.fail_fields(("a", c_typ, 0)) - assert result == (ValueError, 'number of bits invalid for bit field') - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - assert sizeof(X) == sizeof(c_typ) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - assert sizeof(X) == sizeof(c_typ) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - assert result == (ValueError, 'number of bits invalid for bit field') - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == 0 - assert X.a1.offset == sizeof(c_short) - assert X.b.offset == sizeof(c_short)*2 - assert X.c.offset == sizeof(c_short)*2 - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - assert sizeof(X) == sizeof(c_short)*3 - assert X.a.offset == sizeof(c_short)*0 - assert X.b.offset == sizeof(c_short)*1 - assert X.c.offset == sizeof(c_short)*2 - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - import traceback - traceback.print_exc() - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name in ("nt", "ce"): - assert sizeof(X) == sizeof(c_int)*2 - else: - assert sizeof(X) == sizeof(c_int) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - assert sizeof(X) == sizeof(c_int)*2 - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - assert sizeof(X) == sizeof(c_byte) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - def test_set_fields_attr(self): - class A(Structure): - pass - A._fields_ = [("a", c_byte), - ("b", c_ubyte)] - - def test_set_fields_attr_bitfields(self): - class A(Structure): - pass - A._fields_ = [("a", POINTER(A)), - ("b", c_ubyte, 4)] - - def test_set_fields_cycle_fails(self): - class A(Structure): - pass - with pytest.raises(AttributeError): - A._fields_ = [("a", A)] +def test_set_fields_cycle_fails(): + class A(Structure): + pass + with pytest.raises(AttributeError): + A._fields_ = [("a", A)] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py b/extra_tests/ctypes_tests/test_buffers.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py rename to extra_tests/ctypes_tests/test_buffers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_buffers.py +++ b/extra_tests/ctypes_tests/test_buffers.py @@ -1,71 +1,33 @@ from ctypes import * -from .support import BaseCTypesTestChecker -class TestStringBuffer(BaseCTypesTestChecker): +def test_buffer(): + b = create_string_buffer(32) + assert len(b) == 32 + assert sizeof(b) == 32 * sizeof(c_char) + assert type(b[0]) is bytes - def test_buffer(self): - b = create_string_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_char) - assert type(b[0]) is str + b = create_string_buffer(b"abc") + assert len(b) == 4 # trailing nul char + assert sizeof(b) == 4 * sizeof(c_char) + assert type(b[0]) is bytes + assert b[0] == b"a" + assert b[:] == b"abc\0" - b = create_string_buffer(b"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is bytes - assert b[0] == b"a" - assert b[:] == b"abc\0" +def test_from_buffer(): + b1 = bytearray(b"abcde") + b = (c_char * 5).from_buffer(b1) + assert b[2] == b"c" + # + b1 = bytearray(b"abcd") + b = c_int.from_buffer(b1) + assert b.value in (1684234849, # little endian + 1633837924) # big endian - def test_string_conversion(self): - b = create_string_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_char) - assert type(b[0]) is str - assert b[0] == "a" - assert b[:] == "abc\0" - - def test_from_buffer(self): - b1 = bytearray("abcde") - b = (c_char * 5).from_buffer(b1) - assert b[2] == "c" - # - b1 = bytearray("abcd") - b = c_int.from_buffer(b1) - assert b.value in (1684234849, # little endian - 1633837924) # big endian - - def test_from_buffer_keepalive(self): - # Issue #2878 - b1 = bytearray("ab") - array = (c_uint16 * 32)() - array[6] = c_uint16.from_buffer(b1) - # this is also what we get on CPython. I don't think it makes - # sense because the array contains just a copy of the number. - assert array._objects == {'6': b1} - - try: - c_wchar - except NameError: - pass - else: - def test_unicode_buffer(self): - b = create_unicode_buffer(32) - assert len(b) == 32 - assert sizeof(b) == 32 * sizeof(c_wchar) - assert type(b[0]) is unicode - - b = create_unicode_buffer(u"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - - def test_unicode_conversion(self): - b = create_unicode_buffer(b"abc") - assert len(b) == 4 # trailing nul char - assert sizeof(b) == 4 * sizeof(c_wchar) - assert type(b[0]) is unicode - assert b[0] == u"a" - assert b[:] == "abc\0" - +def test_from_buffer_keepalive(): + # Issue #2878 + b1 = bytearray(b"ab") + array = (c_uint16 * 32)() + array[6] = c_uint16.from_buffer(b1) + # this is also what we get on CPython. I don't think it makes + # sense because the array contains just a copy of the number. + assert array._objects == {'6': b1} diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py b/extra_tests/ctypes_tests/test_callback_traceback.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py rename to extra_tests/ctypes_tests/test_callback_traceback.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callback_traceback.py +++ b/extra_tests/ctypes_tests/test_callback_traceback.py @@ -1,80 +1,35 @@ # derived from test_random_things.py -import py +import pytest + from ctypes import * -import sys -def callback_func(arg): - 42 / arg - raise ValueError(arg) +_rawffi = pytest.importorskip('_rawffi') -class TestCallbackTraceback: - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. +# +# This test makes sure the exception types *and* the exception +# value is printed correctly. + + at pytest.mark.skipif("sys.flags.inspect") +def test_SystemExit(monkeypatch, capsys): + """ + When an exception is raised in a ctypes callback function, the C + code prints a traceback. When SystemExit is raised, the interpreter + normally exits immediately. + """ + def callback_func(arg): + raise SystemExit(42) + def custom_exit(value): + raise Exception("<<>>" % (value,)) + monkeypatch.setattr(_rawffi, 'exit', custom_exit) + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) + out, err = capsys.readouterr() + assert not err + cb2(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead a additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import StringIO - old_stderr = sys.stderr - logger = sys.stderr = StringIO.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() - - def test_ValueError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - assert out.splitlines()[-1] == ( - "ValueError: 42") - - def test_IntegerDivisionError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_FloatDivisionError(self): - cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - assert out.splitlines()[-1][:19] == ( - "ZeroDivisionError: ") - - def test_TypeErrorDivisionError(self): - cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, "spam") - assert out.splitlines()[-1].startswith( - "TypeError: " - "unsupported operand type(s) for") - - def test_SystemExit(self): - import _rawffi - if sys.flags.inspect: - skip("requires sys.flags.inspect == 0") - def callback_func(arg): - raise SystemExit(42) - def custom_exit(value): - raise Exception("<<>>" % (value,)) - original_exit = _rawffi.exit - try: - _rawffi.exit = custom_exit - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - cb2 = cast(cast(cb, c_void_p), CFUNCTYPE(c_int, c_int)) - out = self.capture_stderr(cb2, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - assert out.splitlines()[-1] == "Exception: <<>>" - # - finally: - _rawffi.exit = original_exit + cb = CFUNCTYPE(c_int, c_int)(callback_func) + cb(0) + out, err = capsys.readouterr() + assert err.splitlines()[-1] == "Exception: <<>>" diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/extra_tests/ctypes_tests/test_callbacks.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py rename to extra_tests/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/extra_tests/ctypes_tests/test_callbacks.py @@ -1,283 +1,199 @@ +import pytest + +import math from ctypes import * -import pytest from .support import BaseCTypesTestChecker -class TestCallbacks(BaseCTypesTestChecker): - functype = CFUNCTYPE - -## def tearDown(self): -## import gc -## gc.collect() - - def callback(self, *args): - self.got_args = args - return args[-1] - - def check_type(self, typ, arg): - unwrapped_types = { - c_float: (float,), - c_double: (float,), - c_char: (str,), - c_char_p: (str,), - c_uint: (int, long), - c_ulong: (int, long), - } - - PROTO = self.functype.im_func(typ, typ) - cfunc = PROTO(self.callback) - result = cfunc(arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (arg,) - assert result == arg - - result2 = cfunc(typ(arg)) - assert type(result2) in unwrapped_types.get(typ, (int, long)) - - PROTO = self.functype.im_func(typ, c_byte, typ) - result = PROTO(self.callback)(-3, arg) - if typ == c_float: - assert abs(result - arg) < 0.000001 - else: - assert self.got_args == (-3, arg) - assert result == arg - - ################ - - def test_byte(self): - self.check_type(c_byte, 42) - self.check_type(c_byte, -42) - - def test_ubyte(self): - self.check_type(c_ubyte, 42) - - def test_short(self): - self.check_type(c_short, 42) - self.check_type(c_short, -42) - - def test_ushort(self): - self.check_type(c_ushort, 42) - - def test_int(self): - self.check_type(c_int, 42) - self.check_type(c_int, -42) - - def test_uint(self): - self.check_type(c_uint, 42) - - def test_long(self): - self.check_type(c_long, 42) - self.check_type(c_long, -42) - - def test_ulong(self): - self.check_type(c_ulong, 42) - - def test_longlong(self): - self.check_type(c_longlong, 42) - self.check_type(c_longlong, -42) - - def test_ulonglong(self): - self.check_type(c_ulonglong, 42) - - def test_float(self): - # only almost equal: double -> float -> double - import math - self.check_type(c_float, math.e) - self.check_type(c_float, -math.e) - - def test_double(self): - self.check_type(c_double, 3.14) - self.check_type(c_double, -3.14) - - def test_char(self): - self.check_type(c_char, "x") - self.check_type(c_char, "a") - - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. -## def test_char_p(self): -## self.check_type(c_char_p, "abc") -## self.check_type(c_char_p, "def") - - - @pytest.mark.xfail( - reason="we are less strict about callback return type sanity") - def test_unsupported_restype_1(self): - # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. - # POINTER(c_double), for example, is not supported. - - prototype = self.functype.im_func(POINTER(c_double)) - # The type is checked when the prototype is called - with pytest.raises(TypeError): - prototype(lambda: None) - +functypes = [CFUNCTYPE] try: - WINFUNCTYPE + functypes.append(WINFUNCTYPE) except NameError: pass -else: - class TestStdcallCallbacks(TestCallbacks): - functype = WINFUNCTYPE -################################################################ -class TestSampleCallbacks(BaseCTypesTestChecker): +def callback(*args): + callback.got_args = args + return args[-1] - def test_integrate(self): - # Derived from some then non-working code, posted by David Foster - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) +unwrapped_types = { + c_float: (float,), + c_double: (float,), + c_char: (bytes,), + c_char_p: (bytes,), + c_uint: (int,), + c_ulong: (int,), + } - # The function prototype called by 'integrate': double func(double); - CALLBACK = CFUNCTYPE(c_double, c_double) + at pytest.mark.parametrize("typ, arg", [ + (c_byte, 42), + (c_byte, -42), + (c_ubyte, 42), + (c_short, 42), + (c_short, -42), + (c_ushort, 42), + (c_int, 42), + (c_int, -42), + (c_uint, 42), + (c_long, 42), + (c_long, -42), + (c_ulong, 42), + (c_longlong, 42), + (c_longlong, -42), + (c_ulonglong, 42), + (c_float, math.e), # only almost equal: double -> float -> double + (c_float, -math.e), + (c_double, 3.14), + (c_double, -3.14), + (c_char, b"x"), + (c_char, b"a"), +]) + at pytest.mark.parametrize('functype', functypes) +def test_types(typ, arg, functype): + PROTO = functype(typ, typ) + cfunc = PROTO(callback) + result = cfunc(arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (arg,) + assert result == arg - # The integrate function itself, exposed from the _ctypes_test dll - integrate = dll.integrate - integrate.argtypes = (c_double, c_double, CALLBACK, c_long) - integrate.restype = c_double + result2 = cfunc(typ(arg)) + assert type(result2) in unwrapped_types.get(typ, (int,)) - def func(x): - print 'calculating x**2 of',x - return x**2 + PROTO = functype(typ, c_byte, typ) + result = PROTO(callback)(-3, arg) + if typ == c_float: + assert abs(result - arg) < 0.000001 + else: + assert callback.got_args == (-3, arg) + assert result == arg - result = integrate(0.0, 1.0, CALLBACK(func), 10) - diff = abs(result - 1./3.) + at pytest.mark.parametrize('functype', functypes) +def test_unsupported_restype_1(functype): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. - assert diff < 0.01, "%s not less than 0.01" % diff + prototype = functype(POINTER(c_double)) + # The type is checked when the prototype is called + with pytest.raises(TypeError): + prototype(lambda: None) -################################################################ -class TestMoreCallbacks(BaseCTypesTestChecker): +def test_callback_with_struct_argument(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - def test_callback_with_struct_argument(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] + proto = CFUNCTYPE(c_int, RECT) - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - point.left *= -1 - return point.left+point.top+point.right+point.bottom + def callback(point): + point.left *= -1 + return point.left + point.top + point.right + point.bottom - cbp = proto(callback) + cbp = proto(callback) + rect = RECT(-1000, 100, 10, 1) + res = cbp(rect) + assert res == 1111 + assert rect.left == -1000 # must not have been changed! - rect = RECT(-1000,100,10,1) +def test_callback_from_c_with_struct_argument(dll): + class RECT(Structure): + _fields_ = [("left", c_long), ("top", c_long), + ("right", c_long), ("bottom", c_long)] - res = cbp(rect) + proto = CFUNCTYPE(c_int, RECT) - assert res == 1111 - assert rect.left == -1000 # must not have been changed! + def callback(point): + return point.left + point.top + point.right + point.bottom - def test_callback_from_c_with_struct_argument(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + cbp = proto(callback) + rect = RECT(1000, 100, 10, 1) - class RECT(Structure): - _fields_ = [("left", c_long), ("top", c_long), - ("right", c_long), ("bottom", c_long)] + call_callback_with_rect = dll.call_callback_with_rect + call_callback_with_rect.restype = c_int + call_callback_with_rect.argtypes = [proto, RECT] + res = call_callback_with_rect(cbp, rect) + assert res == 1111 - proto = CFUNCTYPE(c_int, RECT) - def callback(point): - return point.left+point.top+point.right+point.bottom +def test_callback_unsupported_return_struct(): + class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] - cbp = proto(callback) - rect = RECT(1000,100,10,1) + proto = CFUNCTYPE(RECT, c_int) + with pytest.raises(TypeError): + proto(lambda r: 0) - call_callback_with_rect = dll.call_callback_with_rect - call_callback_with_rect.restype = c_int - call_callback_with_rect.argtypes = [proto, RECT] - res = call_callback_with_rect(cbp, rect) - assert res == 1111 - def test_callback_unsupported_return_struct(self): - class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] +def test_qsort(dll): + PI = POINTER(c_int) + A = c_int*5 + a = A() + for i in range(5): + a[i] = 5-i - proto = CFUNCTYPE(RECT, c_int) - with pytest.raises(TypeError): - proto(lambda r: 0) + assert a[0] == 5 # sanity + def comp(a, b): + a = a.contents.value + b = b.contents.value + if a < b: + return -1 + elif a > b: + return 1 + else: + return 0 + qs = dll.my_qsort + qs.restype = None + CMP = CFUNCTYPE(c_int, PI, PI) + qs.argtypes = (PI, c_size_t, c_size_t, CMP) - def test_qsort(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) + qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) - PI = POINTER(c_int) - A = c_int*5 - a = A() - for i in range(5): - a[i] = 5-i + res = list(a) - assert a[0] == 5 # sanity + assert res == [1,2,3,4,5] - def comp(a, b): - a = a.contents.value - b = b.contents.value - return cmp(a,b) - qs = dll.my_qsort - qs.restype = None - CMP = CFUNCTYPE(c_int, PI, PI) - qs.argtypes = (PI, c_size_t, c_size_t, CMP) +def test_pyobject_as_opaque(dll): + def callback(arg): + return arg() - qs(cast(a, PI), 5, sizeof(c_int), CMP(comp)) + CTP = CFUNCTYPE(c_int, py_object) + cfunc = dll._testfunc_callback_opaque + cfunc.argtypes = [CTP, py_object] + cfunc.restype = c_int + res = cfunc(CTP(callback), lambda : 3) + assert res == 3 - res = list(a) +def test_callback_void(capsys, dll): + def callback(): + pass - assert res == [1,2,3,4,5] + CTP = CFUNCTYPE(None) + cfunc = dll._testfunc_callback_void + cfunc.argtypes = [CTP] + cfunc.restype = int + cfunc(CTP(callback)) + out, err = capsys.readouterr() + assert (out, err) == ("", "") - def test_pyobject_as_opaque(self): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - def callback(arg): - return arg() +def test_callback_pyobject(): + def callback(obj): + return obj - CTP = CFUNCTYPE(c_int, py_object) - cfunc = dll._testfunc_callback_opaque - cfunc.argtypes = [CTP, py_object] - cfunc.restype = c_int - res = cfunc(CTP(callback), lambda : 3) - assert res == 3 + FUNC = CFUNCTYPE(py_object, py_object) + cfunc = FUNC(callback) + param = c_int(42) + assert cfunc(param) is param - def test_callback_void(self, capsys): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - - def callback(): - pass - - CTP = CFUNCTYPE(None) - cfunc = dll._testfunc_callback_void - cfunc.argtypes = [CTP] - cfunc.restype = int - cfunc(CTP(callback)) - out, err = capsys.readouterr() - assert (out, err) == ("", "") - - - def test_callback_pyobject(self): - def callback(obj): - return obj - - FUNC = CFUNCTYPE(py_object, py_object) - cfunc = FUNC(callback) - param = c_int(42) - assert cfunc(param) is param - - def test_raise_argumenterror(self): - def callback(x): - pass - FUNC = CFUNCTYPE(None, c_void_p) - cfunc = FUNC(callback) - param = c_uint(42) - with pytest.raises(ArgumentError): - cfunc(param) +def test_raise_argumenterror(): + def callback(x): + pass + FUNC = CFUNCTYPE(None, c_void_p) + cfunc = FUNC(callback) + param = c_uint(42) + with pytest.raises(ArgumentError): + cfunc(param) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/extra_tests/ctypes_tests/test_cast.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_cast.py rename to extra_tests/ctypes_tests/test_cast.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py +++ b/extra_tests/ctypes_tests/test_cast.py @@ -1,106 +1,30 @@ +import pytest + from ctypes import * -import sys, py -from .support import BaseCTypesTestChecker -def setup_module(mod): - import conftest - mod.lib = CDLL(str(conftest.sofile)) +def test_cast_functype(dll): + # make sure we can cast function type + my_sqrt = dll.my_sqrt + saved_objects = my_sqrt._objects.copy() + sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) + assert sqrt(4.0) == 2.0 + assert not cast(0, CFUNCTYPE(c_int)) + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) -class TestCast(BaseCTypesTestChecker): +def test_cast_argumenterror(): + param = c_uint(42) + with pytest.raises(ArgumentError): + cast(param, c_void_p) - def test_array2pointer(self): - array = (c_int * 3)(42, 17, 2) - - # casting an array to a pointer works. - ptr = cast(array, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - if 2*sizeof(c_short) == sizeof(c_int): - ptr = cast(array, POINTER(c_short)) - if sys.byteorder == "little": - assert [ptr[i] for i in range(6)] == ( - [42, 0, 17, 0, 2, 0]) - else: - assert [ptr[i] for i in range(6)] == ( - [0, 42, 0, 17, 0, 2]) - - def test_address2pointer(self): - array = (c_int * 3)(42, 17, 2) - - address = addressof(array) - ptr = cast(c_void_p(address), POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - ptr = cast(address, POINTER(c_int)) - assert [ptr[i] for i in range(3)] == [42, 17, 2] - - def test_p2a_objects(self): - py.test.skip("we make copies of strings") - array = (c_char_p * 5)() - assert array._objects is None - array[0] = "foo bar" - assert array._objects == {'0': "foo bar"} - - p = cast(array, POINTER(c_char_p)) - # array and p share a common _objects attribute - assert p._objects is array._objects - assert array._objects == {'0': "foo bar", id(array): array} - p[0] = "spam spam" - assert p._objects == {'0': "spam spam", id(array): array} - assert array._objects is p._objects - p[1] = "foo bar" - assert p._objects == {'1': 'foo bar', '0': "spam spam", id(array): array} - assert array._objects is p._objects - - def test_other(self): - p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) - assert p[:4] == [1,2, 3, 4] - c_int() - assert p[:4] == [1, 2, 3, 4] - p[2] = 96 - assert p[:4] == [1, 2, 96, 4] - c_int() - assert p[:4] == [1, 2, 96, 4] - - def test_char_p(self): - # This didn't work: bad argument to internal function - s = c_char_p("hiho") - - assert cast(cast(s, c_void_p), c_char_p).value == ( - "hiho") - - try: - c_wchar_p - except NameError: - pass - else: - def test_wchar_p(self): - s = c_wchar_p("hiho") - assert cast(cast(s, c_void_p), c_wchar_p).value == ( - "hiho") - - def test_cast_functype(self): - # make sure we can cast function type - my_sqrt = lib.my_sqrt - saved_objects = my_sqrt._objects.copy() - sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) - assert sqrt(4.0) == 2.0 - assert not cast(0, CFUNCTYPE(c_int)) - # - assert sqrt._objects is my_sqrt._objects # on CPython too - my_sqrt._objects.clear() - my_sqrt._objects.update(saved_objects) - - def test_cast_argumenterror(self): - param = c_uint(42) - py.test.raises(ArgumentError, "cast(param, c_void_p)") - - def test_c_bool(self): - x = c_bool(42) - assert x.value is True - x = c_bool(0.0) - assert x.value is False - x = c_bool("") - assert x.value is False - x = c_bool(['yadda']) - assert x.value is True +def test_c_bool(): + x = c_bool(42) + assert x.value is True + x = c_bool(0.0) + assert x.value is False + x = c_bool("") + assert x.value is False + x = c_bool(['yadda']) + assert x.value is True diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py b/extra_tests/ctypes_tests/test_commethods.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_commethods.py rename to extra_tests/ctypes_tests/test_commethods.py diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/extra_tests/ctypes_tests/test_errno.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_errno.py rename to extra_tests/ctypes_tests/test_errno.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py +++ b/extra_tests/ctypes_tests/test_errno.py @@ -1,26 +1,19 @@ -import py +import pytest import ctypes -from _ctypes import function +_rawffi = pytest.importorskip('_rawffi') # PyPy-only -try: - import _rawffi -except ImportError: - py.test.skip("app-level test only for PyPy") +def test_errno_saved_and_restored(): + def check(): + assert _rawffi.get_errno() == 42 + assert ctypes.get_errno() == old + check.free_temp_buffers = lambda *args: None + f = ctypes._CFuncPtr() + old = _rawffi.get_errno() + f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO + ctypes.set_errno(42) + f._call_funcptr(check) + assert _rawffi.get_errno() == old + ctypes.set_errno(0) -class TestErrno: - - def test_errno_saved_and_restored(self): - def check(): - assert _rawffi.get_errno() == 42 - assert ctypes.get_errno() == old - check.free_temp_buffers = lambda *args: None - f = function.CFuncPtr() - old = _rawffi.get_errno() - f._flags_ = _rawffi.FUNCFLAG_USE_ERRNO - ctypes.set_errno(42) - f._call_funcptr(check) - assert _rawffi.get_errno() == old - ctypes.set_errno(0) - - # see also test_functions.test_errno +# see also test_functions.test_errno diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py b/extra_tests/ctypes_tests/test_extra.py rename from pypy/module/test_lib_pypy/ctypes_tests/test_extra.py rename to extra_tests/ctypes_tests/test_extra.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_extra.py +++ b/extra_tests/ctypes_tests/test_extra.py @@ -5,244 +5,239 @@ import py from ctypes import * -from .support import BaseCTypesTestChecker -class TestExtra(BaseCTypesTestChecker): - def test_primitive_pointer(self): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 +def test_primitive_pointer(): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 - ## *** the rest is commented out because it should work but occasionally - ## *** trigger a ctypes bug (SourceForge bug #1467852). *** - ## q = pointer(y) - ## pp[0] = q # pp ---> p ---> y = 14 - ## assert y.value == 14 # (^^^ not q! ) - ## assert p.contents.value == 14 - ## assert pp.contents.contents.value == 14 - ## q.contents = x - ## assert pp.contents.contents.value == 14 +## *** the rest is commented out because it should work but occasionally +## *** trigger a ctypes bug (SourceForge bug #1467852). *** +## q = pointer(y) +## pp[0] = q # pp ---> p ---> y = 14 +## assert y.value == 14 # (^^^ not q! ) +## assert p.contents.value == 14 +## assert pp.contents.contents.value == 14 +## q.contents = x +## assert pp.contents.contents.value == 14 - def test_char_p(self): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" +def test_char_p(): + x = c_char_p(b"hello\x00world") + assert x.value == b"hello" + x.value = b"world" + assert x.value == b"world" - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" + p = pointer(x) + assert p[0] == x.value == b"world" + p[0] = b"other" + assert x.value == p.contents.value == p[0] == b"other" - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" + myarray = (c_char_p * 10)() + myarray[7] = b"hello" + assert isinstance(myarray[7], bytes) + assert myarray[7] == b"hello" - def test_struct(self): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] +def test_struct(): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 - def test_ptr_array(self): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) +def test_ptr_array(): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 - def test_void_p(self): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 +def test_void_p(): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 - def test_char_array(self): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' +def test_char_array(): + a = (c_char * 3)() + a[0] = b'x' + a[1] = b'y' + assert a.value == b'xy' + a[2] = b'z' + assert a.value == b'xyz' - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' + b.value = b"nxw" + assert b[0] == b'n' + assert b[1] == b'x' + assert b[2] == b'w' - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' + b.value = b"?" + assert b[0] == b'?' + assert b[1] == b'\x00' + assert b[2] == b'w' - class S(Structure): - _fields_ = [('p', POINTER(c_char))] + class S(Structure): + _fields_ = [('p', POINTER(c_char))] - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' + s = S() + s.p = b + s.p.contents.value = b'!' + assert b.value == b'!' - assert len(create_string_buffer(0)) == 0 + assert len(create_string_buffer(0)) == 0 - def test_array(self): - a = (c_int * 10)() +def test_array(): + a = (c_int * 10)() - class S(Structure): - _fields_ = [('p', POINTER(c_int))] + class S(Structure): + _fields_ = [('p', POINTER(c_int))] - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] - def test_truth_value(self): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... +def test_truth_value(): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) - assert not c_double(0.0) - assert not c_ulonglong(0) - assert c_ulonglong(2**42) + assert c_int(12) From pypy.commits at gmail.com Sun Dec 16 12:08:30 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 16 Dec 2018 09:08:30 -0800 (PST) Subject: [pypy-commit] pypy default: Skip pyrepl tests on CPython if it's not available Message-ID: <5c16868e.1c69fb81.fa8e.5b1d@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95504:0df22478e494 Date: 2018-12-16 17:07 +0000 http://bitbucket.org/pypy/pypy/changeset/0df22478e494/ Log: Skip pyrepl tests on CPython if it's not available diff --git a/extra_tests/test_pyrepl/conftest.py b/extra_tests/test_pyrepl/conftest.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/conftest.py @@ -0,0 +1,8 @@ +import sys + +def pytest_ignore_collect(path): + if '__pypy__' not in sys.builtin_module_names: + try: + import pyrepl + except ImportError: + return True From pypy.commits at gmail.com Mon Dec 17 04:41:50 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 17 Dec 2018 01:41:50 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: slightly minimize diff to unicode-utf8 Message-ID: <5c176f5e.1c69fb81.547c0.8eda@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95505:7c80f1aec309 Date: 2018-12-17 09:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7c80f1aec309/ Log: slightly minimize diff to unicode-utf8 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -116,6 +116,43 @@ assert len(s) == 4 assert s == u'a\xe9\x00c' + def test_default_encoded_string(self): + import sys + module = self.import_extension('foo', [ + ("test_default_encoded_string", "METH_O", + ''' + PyObject* result = _PyUnicode_AsDefaultEncodedString(args, "replace"); + Py_INCREF(result); + return result; + ''' + ), + ]) + res = module.test_default_encoded_string(u"xyz") + assert res == b'xyz' + res = module.test_default_encoded_string(u"caf\xe9") + assert res == u"caf\xe9".encode(sys.getdefaultencoding(), 'replace') + + def test_unicode_macros(self): + """The PyUnicode_* macros cast, and calls expecting that build.""" + module = self.import_extension('foo', [ + ("test_macro_invocations", "METH_NOARGS", + """ + PyObject* o = PyUnicode_FromString(""); + PyUnicodeObject* u = (PyUnicodeObject*)o; + + PyUnicode_GET_SIZE(u); + PyUnicode_GET_SIZE(o); + + PyUnicode_GET_DATA_SIZE(u); + PyUnicode_GET_DATA_SIZE(o); + + PyUnicode_AS_UNICODE(o); + PyUnicode_AS_UNICODE(u); + return o; + """)]) + assert module.test_macro_invocations() == u'' + + def test_format_v(self): module = self.import_extension('foo', [ ("test_unicode_format_v", "METH_VARARGS", @@ -139,7 +176,7 @@ res = module.test_unicode_format_v(1, "xyz") assert res == "bla 1 ble xyz\n" - def test_format(self): + def test_format1(self): module = self.import_extension('foo', [ ("test_unicode_format", "METH_VARARGS", ''' @@ -273,27 +310,6 @@ assert module.compare("", b"abc") == -1 assert module.compare("abc", b"") == 1 - - def test_unicode_macros(self): - """The PyUnicode_* macros cast, and calls expecting that build.""" - module = self.import_extension('foo', [ - ("test_macro_invocations", "METH_NOARGS", - """ - PyObject* o = PyUnicode_FromString(""); - PyUnicodeObject* u = (PyUnicodeObject*)o; - - PyUnicode_GET_SIZE(u); - PyUnicode_GET_SIZE(o); - - PyUnicode_GET_DATA_SIZE(u); - PyUnicode_GET_DATA_SIZE(o); - - PyUnicode_AS_UNICODE(o); - PyUnicode_AS_UNICODE(u); - return o; - """)]) - assert module.test_macro_invocations() == u'' - def test_AsUTF8AndSize(self): module = self.import_extension('foo', [ ("utf8", "METH_O", @@ -886,19 +902,19 @@ def test_fromordinal(self, space): w_char = PyUnicode_FromOrdinal(space, 65) - assert space.utf8_w(w_char) == 'A' + assert space.utf8_w(w_char) == u'A' w_char = PyUnicode_FromOrdinal(space, 0) - assert space.utf8_w(w_char) == '\0' + assert space.utf8_w(w_char) == u'\0' w_char = PyUnicode_FromOrdinal(space, 0xFFFF) - assert space.utf8_w(w_char) == u'\uFFFF'.encode("utf-8") + assert space.utf8_w(w_char) == u'\uFFFF'.encode('utf-8') def test_replace(self, space): w_str = space.wrap(u"abababab") w_substr = space.wrap(u"a") w_replstr = space.wrap(u"z") - assert "zbzbabab" == space.utf8_w( + assert u"zbzbabab" == space.utf8_w( PyUnicode_Replace(space, w_str, w_substr, w_replstr, 2)) - assert "zbzbzbzb" == space.utf8_w( + assert u"zbzbzbzb" == space.utf8_w( PyUnicode_Replace(space, w_str, w_substr, w_replstr, -1)) def test_tailmatch(self, space): From pypy.commits at gmail.com Mon Dec 17 04:41:52 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 17 Dec 2018 01:41:52 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix some datetime.py test failures. Should this file move to lib_pypy? Message-ID: <5c176f60.1c69fb81.2d293.4256@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95506:b5e988c47134 Date: 2018-12-17 11:40 +0200 http://bitbucket.org/pypy/pypy/changeset/b5e988c47134/ Log: fix some datetime.py test failures. Should this file move to lib_pypy? diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -521,7 +521,11 @@ -self._microseconds) def __pos__(self): - return self + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days, + self._seconds, + self._microseconds) def __abs__(self): if self._days < 0: @@ -813,8 +817,7 @@ month = self._month if day is None: day = self._day - # PyPy fix: returns type(self)() instead of date() - return type(self)(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1289,8 +1292,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - # PyPy fix: returns type(self)() instead of time() - return type(self)(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) # Pickle support. @@ -1341,13 +1344,13 @@ hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = dateinterop.__new__(cls) - self._year = year - self._month = month - self._day = day - self._hour = hour - self._minute = minute - self._second = second - self._microsecond = microsecond + self._year = int(year) + self._month = int(month) + self._day = int(day) + self._hour = int(hour) + self._minute = int(minute) + self._second = int(second) + self._microsecond = int(microsecond) self._tzinfo = tzinfo self._hashcode = -1 return self @@ -1503,8 +1506,8 @@ if tzinfo is True: tzinfo = self.tzinfo # PyPy fix: returns type(self)() instead of datetime() - return type(self)(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), year, month, day, hour, minute, + second, microsecond, tzinfo) def astimezone(self, tz=None): if tz is None: @@ -1768,7 +1771,10 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + # The CPython _datetimemodule.c error message and the + # datetime.py one are different + raise TypeError("can't subtract offset-naive and " + "offset-aware datetimes") return base + otoff - myoff def __hash__(self): From pypy.commits at gmail.com Mon Dec 17 12:02:04 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 17 Dec 2018 09:02:04 -0800 (PST) Subject: [pypy-commit] pypy default: rm compat code for CPython<=2.5 Message-ID: <5c17d68c.1c69fb81.a9341.71c2@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95507:7eccdb844fb8 Date: 2018-12-17 17:01 +0000 http://bitbucket.org/pypy/pypy/changeset/7eccdb844fb8/ Log: rm compat code for CPython<=2.5 diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -6,11 +6,6 @@ from rpython.rtyper.lltypesystem.module.test.math_cases import (MathTests, get_tester) -consistent_host = True -if '__pypy__' not in sys.builtin_module_names: - if sys.version_info < (2, 6): - consistent_host = False - class TestDirect(MathTests): pass @@ -30,8 +25,6 @@ def make_test_case((fnname, args, expected), dict): # def test_func(self): - if not consistent_host: - py.test.skip("inconsistent behavior before 2.6") try: fn = getattr(math, fnname) except AttributeError: diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -24,7 +24,6 @@ expected = space.wrap(expected) cases.append(space.newtuple([space.wrap(a), space.wrap(b), expected])) cls.w_cases = space.newlist(cases) - cls.w_consistent_host = space.wrap(test_direct.consistent_host) @classmethod def make_callable_wrapper(cls, func): @@ -36,8 +35,6 @@ assert abs(actual - expected) < 10E-5 def test_all_cases(self): - if not self.consistent_host: - skip("please test this on top of PyPy or CPython >= 2.6") import math for fnname, args, expected in self.cases: fn = getattr(math, fnname) From pypy.commits at gmail.com Tue Dec 18 11:10:57 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 18 Dec 2018 08:10:57 -0800 (PST) Subject: [pypy-commit] cffi default: Fix (very unlikely case though) Message-ID: <5c191c11.1c69fb81.d8457.b18b@mx.google.com> Author: Armin Rigo Branch: Changeset: r3176:4e1b8c2f5410 Date: 2018-12-18 17:10 +0100 http://bitbucket.org/cffi/cffi/changeset/4e1b8c2f5410/ Log: Fix (very unlikely case though) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3489,6 +3489,7 @@ if (ct->ct_flags & CT_WITH_VAR_ARRAY) { PyErr_SetString(PyExc_TypeError, "return type is a struct/union with a varsize array member"); + return NULL; } cd = allocate_owning_object(dataoffset + datasize, ct, /*dont_clear=*/1); if (cd == NULL) From pypy.commits at gmail.com Wed Dec 19 06:00:30 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 03:00:30 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: encode the gc states in 16 bit instead of 32, to avoid messing with unsigned constants on 32 bit Message-ID: <5c1a24ce.1c69fb81.d66e8.ff93@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95508:96498bace39f Date: 2018-12-19 11:59 +0100 http://bitbucket.org/pypy/pypy/changeset/96498bace39f/ Log: encode the gc states in 16 bit instead of 32, to avoid messing with unsigned constants on 32 bit diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -31,13 +31,13 @@ return _encode_states(1, 0) def _encode_states(oldstate, newstate): - return oldstate << 16 | newstate + return oldstate << 8 | newstate def old_state(states): - return (states & 0xFFFF0000) >> 16 + return (states & 0xFF00) >> 8 def new_state(states): - return states & 0xFFFF + return states & 0xFF def is_done(states): """ From pypy.commits at gmail.com Wed Dec 19 10:01:52 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 07:01:52 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: write docs for the new GC API Message-ID: <5c1a5d60.1c69fb81.68f7a.9695@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95509:9b7167f7a2ac Date: 2018-12-19 16:00 +0100 http://bitbucket.org/pypy/pypy/changeset/9b7167f7a2ac/ Log: write docs for the new GC API diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -395,7 +395,8 @@ * some functions and attributes of the ``gc`` module behave in a slightly different way: for example, ``gc.enable`` and ``gc.disable`` are supported, but instead of enabling and disabling - the GC, they just enable and disable the execution of finalizers. + the GC, they just enable and disable the major collections and the + execution of finalizers. * PyPy prints a random line from past #pypy IRC topics at startup in interactive mode. In a released version, this behaviour is suppressed, but diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -22,8 +22,44 @@ larger. (A third category, the very large objects, are initially allocated outside the nursery and never move.) -Since Incminimark is an incremental GC, the major collection is incremental, -meaning there should not be any pauses longer than 1ms. +Since Incminimark is an incremental GC, the major collection is incremental: +the goal is not to have any pause longer than 1ms, but in practice it depends +on the size and characteristics of the heap: occasionally, there can be pauses +between 10-100ms. + + +Semi-manual GC management +-------------------------- + +If there are parts of the program where it is important to have a low latency, +you might want to control precisely when the GC runs, to avoid unexpected +pauses. Note that this has effect only on major collections, while minor +collections continue to work as usual. + +As explained above, a full major collection consists of ``N`` steps, where +``N`` depends on the size of the heap; generally speaking, it is not possible +to predict how many steps will be needed to complete a collection. + +``gc.enable()`` and ``gc.disable()`` control whether the GC runs collection +steps automatically. When the GC is disabled the memory usage will grow +indefinitely, unless you manually call ``gc.collect()`` and +``gc.collect_step()``. + +``gc.collect()`` runs a full major collection. + +``gc.collect_step()`` runs a single collection step. It returns an object of +type GcCollectStepStats_, the same which is passed to the corresponding `GC +Hooks`_. The following code is roughly equivalent to a ``gc.collect()``:: + + while True: + if gc.collect_step().major_is_done: + break + +For a real-world example of usage of this API, you can look at the 3rd-party +module `pypytools.gc.custom`_, which also provides a ``with customgc.nogc()`` +context manager to mark sections where the GC is forbidden. + +.. _`pypytools.gc.custom`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default Fragmentation @@ -184,6 +220,8 @@ the number of pinned objects. +.. _GcCollectStepStats: + The attributes for ``GcCollectStepStats`` are: ``count``, ``duration``, ``duration_min``, ``duration_max`` @@ -192,10 +230,14 @@ ``oldstate``, ``newstate`` Integers which indicate the state of the GC before and after the step. +``major_is_done`` + Boolean which indicate whether this was the last step of the major + collection + The value of ``oldstate`` and ``newstate`` is one of these constants, defined inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, -``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string -representation of it by indexing the ``GC_STATS`` tuple. +``STATE_SWEEPING``, ``STATE_FINALIZING``, ``STATE_USERDEL``. It is possible +to get a string representation of it by indexing the ``GC_STATES`` tuple. The attributes for ``GcCollectStats`` are: From pypy.commits at gmail.com Wed Dec 19 10:01:54 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 07:01:54 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: fix whatsnew Message-ID: <5c1a5d62.1c69fb81.9553.3f67@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95510:9901aada1e1b Date: 2018-12-19 16:01 +0100 http://bitbucket.org/pypy/pypy/changeset/9901aada1e1b/ Log: fix whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -56,3 +56,8 @@ .. branch: expose-gc-time Make GC hooks measure time in seconds (as opposed to an opaque unit). + +.. branch: gc-disable + +Make it possible to manually manage the GC by using a combination of +gc.disable() and gc.collect_step() From pypy.commits at gmail.com Wed Dec 19 11:25:55 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 08:25:55 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: add a note for the next release announcement Message-ID: <5c1a7113.1c69fb81.c47f0.377a@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95511:0a8bccb474c9 Date: 2018-12-19 17:25 +0100 http://bitbucket.org/pypy/pypy/changeset/0a8bccb474c9/ Log: add a note for the next release announcement diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -60,4 +60,6 @@ .. branch: gc-disable Make it possible to manually manage the GC by using a combination of -gc.disable() and gc.collect_step() +gc.disable() and gc.collect_step(). Make sure to write a proper release +announcement in which we explain that existing programs could leak memory if +they run for too much time between a gc.disable()/gc.enable() From pypy.commits at gmail.com Wed Dec 19 12:05:51 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 19 Dec 2018 09:05:51 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: Fix comments and tweaks to the logic in incminimark.collect() Message-ID: <5c1a7a6f.1c69fb81.1cc5f.6cec@mx.google.com> Author: Armin Rigo Branch: gc-disable Changeset: r95512:432d816c6d7b Date: 2018-12-19 18:05 +0100 http://bitbucket.org/pypy/pypy/changeset/432d816c6d7b/ Log: Fix comments and tweaks to the logic in incminimark.collect() diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -394,8 +394,9 @@ * some functions and attributes of the ``gc`` module behave in a slightly different way: for example, ``gc.enable`` and - ``gc.disable`` are supported, but instead of enabling and disabling - the GC, they just enable and disable the major collections and the + ``gc.disable`` are supported, but "enabling and disabling the GC" has + a different meaning in PyPy than in CPython. These functions + actually enable and disable the major collections and the execution of finalizers. * PyPy prints a random line from past #pypy IRC topics at startup in diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -763,13 +763,27 @@ def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full major (gen>=2) collection.""" - if gen <= 0: - self._minor_collection() # dangerous! no major GC cycle progress - elif gen <= 1: + if gen < 0: + # Dangerous! this makes no progress on the major GC cycle. + # If called too often, the memory usage will keep increasing, + # because we'll never completely fill the nursery (and so + # never run anything about the major collection). + self._minor_collection() + elif gen == 0: + # This runs a minor collection. This is basically what occurs + # when the nursery is full. If a major collection is in + # progress, it also runs one more step of it. It might also + # decide to start a major collection just now, depending on + # current memory pressure. self.minor_collection_with_major_progress(force_enabled=True) - if gen == 1 and self.gc_state == STATE_SCANNING: + elif gen == 1: + # This is like gen == 0, but if no major collection is running, + # then it forces one to start now. + self.minor_collection_with_major_progress(force_enabled=True) + if self.gc_state == STATE_SCANNING: self.major_collection_step() else: + # This does a complete minor and major collection. self.minor_and_major_collection() self.rrc_invoke_callback() From pypy.commits at gmail.com Wed Dec 19 12:15:16 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 09:15:16 -0800 (PST) Subject: [pypy-commit] pypy gc-disable: close merged branch Message-ID: <5c1a7ca4.1c69fb81.fa0a6.8231@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95513:e6dcaa99d7ab Date: 2018-12-19 18:11 +0100 http://bitbucket.org/pypy/pypy/changeset/e6dcaa99d7ab/ Log: close merged branch From pypy.commits at gmail.com Wed Dec 19 12:15:22 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 19 Dec 2018 09:15:22 -0800 (PST) Subject: [pypy-commit] pypy default: Merge the gc-disable branch, which introduces two changes to the gc module: Message-ID: <5c1a7caa.1c69fb81.d7041.0a5b@mx.google.com> Author: Antonio Cuni Branch: Changeset: r95514:947a6274ef5e Date: 2018-12-19 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/947a6274ef5e/ Log: Merge the gc-disable branch, which introduces two changes to the gc module: 1. gc.disable() actually disables the major collections of the GC (thus causing an apparent leak if you leave it disabled for too much) 2. gc.collect_step() lets you to manually run collection steps when the GC is disabled diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -394,8 +394,10 @@ * some functions and attributes of the ``gc`` module behave in a slightly different way: for example, ``gc.enable`` and - ``gc.disable`` are supported, but instead of enabling and disabling - the GC, they just enable and disable the execution of finalizers. + ``gc.disable`` are supported, but "enabling and disabling the GC" has + a different meaning in PyPy than in CPython. These functions + actually enable and disable the major collections and the + execution of finalizers. * PyPy prints a random line from past #pypy IRC topics at startup in interactive mode. In a released version, this behaviour is suppressed, but diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -22,8 +22,44 @@ larger. (A third category, the very large objects, are initially allocated outside the nursery and never move.) -Since Incminimark is an incremental GC, the major collection is incremental, -meaning there should not be any pauses longer than 1ms. +Since Incminimark is an incremental GC, the major collection is incremental: +the goal is not to have any pause longer than 1ms, but in practice it depends +on the size and characteristics of the heap: occasionally, there can be pauses +between 10-100ms. + + +Semi-manual GC management +-------------------------- + +If there are parts of the program where it is important to have a low latency, +you might want to control precisely when the GC runs, to avoid unexpected +pauses. Note that this has effect only on major collections, while minor +collections continue to work as usual. + +As explained above, a full major collection consists of ``N`` steps, where +``N`` depends on the size of the heap; generally speaking, it is not possible +to predict how many steps will be needed to complete a collection. + +``gc.enable()`` and ``gc.disable()`` control whether the GC runs collection +steps automatically. When the GC is disabled the memory usage will grow +indefinitely, unless you manually call ``gc.collect()`` and +``gc.collect_step()``. + +``gc.collect()`` runs a full major collection. + +``gc.collect_step()`` runs a single collection step. It returns an object of +type GcCollectStepStats_, the same which is passed to the corresponding `GC +Hooks`_. The following code is roughly equivalent to a ``gc.collect()``:: + + while True: + if gc.collect_step().major_is_done: + break + +For a real-world example of usage of this API, you can look at the 3rd-party +module `pypytools.gc.custom`_, which also provides a ``with customgc.nogc()`` +context manager to mark sections where the GC is forbidden. + +.. _`pypytools.gc.custom`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default Fragmentation @@ -184,6 +220,8 @@ the number of pinned objects. +.. _GcCollectStepStats: + The attributes for ``GcCollectStepStats`` are: ``count``, ``duration``, ``duration_min``, ``duration_max`` @@ -192,10 +230,14 @@ ``oldstate``, ``newstate`` Integers which indicate the state of the GC before and after the step. +``major_is_done`` + Boolean which indicate whether this was the last step of the major + collection + The value of ``oldstate`` and ``newstate`` is one of these constants, defined inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, -``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string -representation of it by indexing the ``GC_STATS`` tuple. +``STATE_SWEEPING``, ``STATE_FINALIZING``, ``STATE_USERDEL``. It is possible +to get a string representation of it by indexing the ``GC_STATES`` tuple. The attributes for ``GcCollectStats`` are: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -60,3 +60,10 @@ .. branch: cleanup-test_lib_pypy Update most test_lib_pypy/ tests and move them to extra_tests/. + +.. branch: gc-disable + +Make it possible to manually manage the GC by using a combination of +gc.disable() and gc.collect_step(). Make sure to write a proper release +announcement in which we explain that existing programs could leak memory if +they run for too much time between a gc.disable()/gc.enable() diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -4,6 +4,7 @@ class Module(MixedModule): interpleveldefs = { 'collect': 'interp_gc.collect', + 'collect_step': 'interp_gc.collect_step', 'enable': 'interp_gc.enable', 'disable': 'interp_gc.disable', 'isenabled': 'interp_gc.isenabled', diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,5 +1,6 @@ from rpython.memory.gc.hook import GcHooks -from rpython.memory.gc import incminimark +from rpython.memory.gc import incminimark +from rpython.rlib import rgc from rpython.rlib.nonconst import NonConstant from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault @@ -117,12 +118,24 @@ self.descr_set_on_gc_collect(space, space.w_None) -class GcMinorHookAction(AsyncAction): +class NoRecursiveAction(AsyncAction): + depth = 0 + + def perform(self, ec, frame): + if self.depth == 0: + try: + self.depth += 1 + return self._do_perform(ec, frame) + finally: + self.depth -= 1 + + +class GcMinorHookAction(NoRecursiveAction): total_memory_used = 0 pinned_objects = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -145,7 +158,7 @@ self.pinned_objects = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcMinorStats( self.count, self.duration, @@ -157,12 +170,12 @@ self.space.call_function(self.w_callable, w_stats) -class GcCollectStepHookAction(AsyncAction): +class GcCollectStepHookAction(NoRecursiveAction): oldstate = 0 newstate = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -185,19 +198,20 @@ self.newstate = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStepStats( self.count, self.duration, self.duration_min, self.duration_max, self.oldstate, - self.newstate) + self.newstate, + rgc.is_done__states(self.oldstate, self.newstate)) self.reset() self.space.call_function(self.w_callable, w_stats) -class GcCollectHookAction(AsyncAction): +class GcCollectHookAction(NoRecursiveAction): num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -206,7 +220,7 @@ rawmalloc_bytes_after = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -227,7 +241,7 @@ self.rawmalloc_bytes_after = NonConstant(r_uint(42)) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStats(self.count, self.num_major_collects, self.arenas_count_before, @@ -252,15 +266,32 @@ class W_GcCollectStepStats(W_Root): + # NOTE: this is specific to incminimark: if we want to integrate the + # applevel gc module with another gc, we probably need a more general + # approach to this. + # + # incminimark has 4 GC states: scanning, marking, sweeping and + # finalizing. However, from the user point of view, we have an additional + # "virtual" state: USERDEL, which represent when we run applevel + # finalizers after having completed a GC major collection. This state is + # never explicitly visible when using hooks, but it is used for the return + # value of gc.collect_step (see interp_gc.py) + STATE_SCANNING = incminimark.STATE_SCANNING + STATE_MARKING = incminimark.STATE_MARKING + STATE_SWEEPING = incminimark.STATE_SWEEPING + STATE_FINALIZING = incminimark.STATE_FINALIZING + STATE_USERDEL = incminimark.STATE_FINALIZING + 1 # used by StepCollector + GC_STATES = tuple(incminimark.GC_STATES + ['USERDEL']) def __init__(self, count, duration, duration_min, duration_max, - oldstate, newstate): + oldstate, newstate, major_is_done): self.count = count self.duration = duration self.duration_min = duration_min self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate + self.major_is_done = major_is_done class W_GcCollectStats(W_Root): @@ -320,11 +351,16 @@ W_GcCollectStepStats.typedef = TypeDef( "GcCollectStepStats", - STATE_SCANNING = incminimark.STATE_SCANNING, - STATE_MARKING = incminimark.STATE_MARKING, - STATE_SWEEPING = incminimark.STATE_SWEEPING, - STATE_FINALIZING = incminimark.STATE_FINALIZING, - GC_STATES = tuple(incminimark.GC_STATES), + STATE_SCANNING = W_GcCollectStepStats.STATE_SCANNING, + STATE_MARKING = W_GcCollectStepStats.STATE_MARKING, + STATE_SWEEPING = W_GcCollectStepStats.STATE_SWEEPING, + STATE_FINALIZING = W_GcCollectStepStats.STATE_FINALIZING, + STATE_USERDEL = W_GcCollectStepStats.STATE_USERDEL, + GC_STATES = tuple(W_GcCollectStepStats.GC_STATES), + major_is_done = interp_attrproperty( + "major_is_done", + cls=W_GcCollectStepStats, + wrapfn="newbool"), **wrap_many(W_GcCollectStepStats, ( "count", "duration", diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import oefmt from rpython.rlib import rgc +from pypy.module.gc.hook import W_GcCollectStepStats @unwrap_spec(generation=int) @@ -16,7 +17,9 @@ cache.clear() rgc.collect() + _run_finalizers(space) +def _run_finalizers(space): # if we are running in gc.disable() mode but gc.collect() is called, # we should still call the finalizers now. We do this as an attempt # to get closer to CPython's behavior: in Py3.5 some tests @@ -39,18 +42,20 @@ return space.newint(0) def enable(space): - """Non-recursive version. Enable finalizers now. + """Non-recursive version. Enable major collections and finalizers. If they were already enabled, no-op. If they were disabled even several times, enable them anyway. """ + rgc.enable() if not space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = True enable_finalizers(space) def disable(space): - """Non-recursive version. Disable finalizers now. Several calls - to this function are ignored. + """Non-recursive version. Disable major collections and finalizers. + Multiple calls to this function are ignored. """ + rgc.disable() if space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = False disable_finalizers(space) @@ -77,6 +82,59 @@ if uda.pending_with_disabled_del is None: uda.pending_with_disabled_del = [] + +class StepCollector(object): + """ + Invoke rgc.collect_step() until we are done, then run the app-level + finalizers as a separate step + """ + + def __init__(self, space): + self.space = space + self.finalizing = False + + def do(self): + if self.finalizing: + self._run_finalizers() + self.finalizing = False + oldstate = W_GcCollectStepStats.STATE_USERDEL + newstate = W_GcCollectStepStats.STATE_SCANNING + major_is_done = True # now we are finally done + else: + states = self._collect_step() + oldstate = rgc.old_state(states) + newstate = rgc.new_state(states) + major_is_done = False # USERDEL still to do + if rgc.is_done(states): + newstate = W_GcCollectStepStats.STATE_USERDEL + self.finalizing = True + # + duration = -1 + return W_GcCollectStepStats( + count = 1, + duration = duration, + duration_min = duration, + duration_max = duration, + oldstate = oldstate, + newstate = newstate, + major_is_done = major_is_done) + + def _collect_step(self): + return rgc.collect_step() + + def _run_finalizers(self): + _run_finalizers(self.space) + +def collect_step(space): + """ + If the GC is incremental, run a single gc-collect-step. Return True when + the major collection is completed. + If the GC is not incremental, do a full collection and return True. + """ + sc = space.fromcache(StepCollector) + w_stats = sc.do() + return w_stats + # ____________________________________________________________ @unwrap_spec(filename='fsencode') diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -1,7 +1,20 @@ import py - +import pytest +from rpython.rlib import rgc +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.module.gc.interp_gc import StepCollector, W_GcCollectStepStats class AppTestGC(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + def rgc_isenabled(space): + return space.newbool(rgc.isenabled()) + cls.w_rgc_isenabled = space.wrap(interp2app(rgc_isenabled)) + def test_collect(self): import gc gc.collect() # mostly a "does not crash" kind of test @@ -63,12 +76,16 @@ def test_enable(self): import gc assert gc.isenabled() + assert self.rgc_isenabled() gc.disable() assert not gc.isenabled() + assert not self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() def test_gc_collect_overrides_gc_disable(self): import gc @@ -83,6 +100,24 @@ assert deleted == [1] gc.enable() + def test_gc_collect_step(self): + import gc + + class X(object): + deleted = 0 + def __del__(self): + X.deleted += 1 + + gc.disable() + X(); X(); X(); + n = 0 + while True: + n += 1 + if gc.collect_step().major_is_done: + break + + assert n >= 2 # at least one step + 1 finalizing + assert X.deleted == 3 class AppTestGcDumpHeap(object): pytestmark = py.test.mark.xfail(run=False) @@ -156,3 +191,55 @@ gc.collect() # the classes C should all go away here for r in rlist: assert r() is None + + +def test_StepCollector(): + W = W_GcCollectStepStats + SCANNING = W.STATE_SCANNING + MARKING = W.STATE_MARKING + SWEEPING = W.STATE_SWEEPING + FINALIZING = W.STATE_FINALIZING + USERDEL = W.STATE_USERDEL + + class MyStepCollector(StepCollector): + my_steps = 0 + my_done = False + my_finalized = 0 + + def __init__(self): + StepCollector.__init__(self, space=None) + self._state_transitions = iter([ + (SCANNING, MARKING), + (MARKING, SWEEPING), + (SWEEPING, FINALIZING), + (FINALIZING, SCANNING)]) + + def _collect_step(self): + self.my_steps += 1 + try: + oldstate, newstate = next(self._state_transitions) + except StopIteration: + assert False, 'should not happen, did you call _collect_step too much?' + return rgc._encode_states(oldstate, newstate) + + def _run_finalizers(self): + self.my_finalized += 1 + + sc = MyStepCollector() + transitions = [] + while True: + result = sc.do() + transitions.append((result.oldstate, result.newstate, sc.my_finalized)) + if result.major_is_done: + break + + assert transitions == [ + (SCANNING, MARKING, False), + (MARKING, SWEEPING, False), + (SWEEPING, FINALIZING, False), + (FINALIZING, USERDEL, False), + (USERDEL, SCANNING, True) + ] + # there is one more transition than actual step, because + # FINALIZING->USERDEL is "virtual" + assert sc.my_steps == len(transitions) - 1 diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -69,26 +69,29 @@ def test_on_gc_collect_step(self): import gc + SCANNING = 0 + MARKING = 1 + SWEEPING = 2 + FINALIZING = 3 lst = [] def on_gc_collect_step(stats): lst.append((stats.count, stats.duration, stats.oldstate, - stats.newstate)) + stats.newstate, + stats.major_is_done)) gc.hooks.on_gc_collect_step = on_gc_collect_step - self.fire_gc_collect_step(10, 20, 30) - self.fire_gc_collect_step(40, 50, 60) + self.fire_gc_collect_step(10, SCANNING, MARKING) + self.fire_gc_collect_step(40, FINALIZING, SCANNING) assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), + (1, 10, SCANNING, MARKING, False), + (1, 40, FINALIZING, SCANNING, True), ] # gc.hooks.on_gc_collect_step = None - self.fire_gc_collect_step(70, 80, 90) # won't fire - assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), - ] + oldlst = lst[:] + self.fire_gc_collect_step(70, SCANNING, MARKING) # won't fire + assert lst == oldlst def test_on_gc_collect(self): import gc @@ -123,7 +126,8 @@ assert S.STATE_MARKING == 1 assert S.STATE_SWEEPING == 2 assert S.STATE_FINALIZING == 3 - assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', + 'FINALIZING', 'USERDEL') def test_cumulative(self): import gc @@ -176,3 +180,22 @@ assert gc.hooks.on_gc_minor is None assert gc.hooks.on_gc_collect_step is None assert gc.hooks.on_gc_collect is None + + def test_no_recursive(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + self.fire_gc_minor(1, 2, 3) # won't fire NOW + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + # the duration for the 2nd call is 41, because it also counts the 1 + # which was fired recursively + assert lst == [ + (1, 10, 20, 30), + (2, 41, 50, 60), + ] diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -149,6 +149,20 @@ def get_size_incl_hash(self, obj): return self.get_size(obj) + # these can be overriden by subclasses, called by the GCTransformer + def enable(self): + pass + + def disable(self): + pass + + def isenabled(self): + return True + + def collect_step(self): + self.collect() + return True + def malloc(self, typeid, length=0, zero=False): """NOT_RPYTHON For testing. The interface used by the gctransformer is diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -379,6 +379,11 @@ self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING + + # if the GC is disabled, it runs only minor collections; major + # collections need to be manually triggered by explicitly calling + # collect() + self.enabled = True # # Two lists of all objects with finalizers. Actually they are lists # of pairs (finalization_queue_nr, object). "probably young objects" @@ -514,6 +519,15 @@ bigobj = self.nonlarge_max + 1 self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + def enable(self): + self.enabled = True + + def disable(self): + self.enabled = False + + def isenabled(self): + return self.enabled + def _nursery_memory_size(self): extra = self.nonlarge_max + 1 return self.nursery_size + extra @@ -750,22 +764,53 @@ """Do a minor (gen=0), start a major (gen=1), or do a full major (gen>=2) collection.""" if gen < 0: - self._minor_collection() # dangerous! no major GC cycle progress - elif gen <= 1: - self.minor_collection_with_major_progress() - if gen == 1 and self.gc_state == STATE_SCANNING: + # Dangerous! this makes no progress on the major GC cycle. + # If called too often, the memory usage will keep increasing, + # because we'll never completely fill the nursery (and so + # never run anything about the major collection). + self._minor_collection() + elif gen == 0: + # This runs a minor collection. This is basically what occurs + # when the nursery is full. If a major collection is in + # progress, it also runs one more step of it. It might also + # decide to start a major collection just now, depending on + # current memory pressure. + self.minor_collection_with_major_progress(force_enabled=True) + elif gen == 1: + # This is like gen == 0, but if no major collection is running, + # then it forces one to start now. + self.minor_collection_with_major_progress(force_enabled=True) + if self.gc_state == STATE_SCANNING: self.major_collection_step() else: + # This does a complete minor and major collection. self.minor_and_major_collection() self.rrc_invoke_callback() + def collect_step(self): + """ + Do a single major collection step. Return True when the major collection + is completed. - def minor_collection_with_major_progress(self, extrasize=0): - """Do a minor collection. Then, if there is already a major GC - in progress, run at least one major collection step. If there is - no major GC but the threshold is reached, start a major GC. + This is meant to be used together with gc.disable(), to have a + fine-grained control on when the GC runs. + """ + old_state = self.gc_state + self._minor_collection() + self.major_collection_step() + self.rrc_invoke_callback() + return rgc._encode_states(old_state, self.gc_state) + + def minor_collection_with_major_progress(self, extrasize=0, + force_enabled=False): + """Do a minor collection. Then, if the GC is enabled and there + is already a major GC in progress, run at least one major collection + step. If there is no major GC but the threshold is reached, start a + major GC. """ self._minor_collection() + if not self.enabled and not force_enabled: + return # If the gc_state is STATE_SCANNING, we're not in the middle # of an incremental major collection. In that case, wait @@ -2428,25 +2473,6 @@ # We also need to reset the GCFLAG_VISITED on prebuilt GC objects. self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None) # - # Print statistics - debug_start("gc-collect-done") - debug_print("arenas: ", - self.stat_ac_arenas_count, " => ", - self.ac.arenas_count) - debug_print("bytes used in arenas: ", - self.ac.total_memory_used) - debug_print("bytes raw-malloced: ", - self.stat_rawmalloced_total_size, " => ", - self.rawmalloced_total_size) - debug_stop("gc-collect-done") - self.hooks.fire_gc_collect( - num_major_collects=self.num_major_collects, - arenas_count_before=self.stat_ac_arenas_count, - arenas_count_after=self.ac.arenas_count, - arenas_bytes=self.ac.total_memory_used, - rawmalloc_bytes_before=self.stat_rawmalloced_total_size, - rawmalloc_bytes_after=self.rawmalloced_total_size) - # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than # we currently have -- but no more than 'max_delta' more than @@ -2460,6 +2486,27 @@ total_memory_used + self.max_delta), reserving_size) # + # Print statistics + debug_start("gc-collect-done") + debug_print("arenas: ", + self.stat_ac_arenas_count, " => ", + self.ac.arenas_count) + debug_print("bytes used in arenas: ", + self.ac.total_memory_used) + debug_print("bytes raw-malloced: ", + self.stat_rawmalloced_total_size, " => ", + self.rawmalloced_total_size) + debug_print("next major collection threshold: ", + self.next_major_collection_threshold) + debug_stop("gc-collect-done") + self.hooks.fire_gc_collect( + num_major_collects=self.num_major_collects, + arenas_count_before=self.stat_ac_arenas_count, + arenas_count_after=self.ac.arenas_count, + arenas_bytes=self.ac.total_memory_used, + rawmalloc_bytes_before=self.stat_rawmalloced_total_size, + rawmalloc_bytes_after=self.rawmalloced_total_size) + # # Max heap size: gives an upper bound on the threshold. If we # already have at least this much allocated, raise MemoryError. if bounded and self.threshold_reached(reserving_size): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -13,6 +13,7 @@ from rpython.memory.gc import minimark, incminimark from rpython.memory.gctypelayout import zero_gc_pointers_inside, zero_gc_pointers from rpython.rlib.debug import debug_print +from rpython.rlib.test.test_debug import debuglog import pdb WORD = LONG_BIT // 8 @@ -770,4 +771,76 @@ assert elem.prev == lltype.nullptr(S) assert elem.next == lltype.nullptr(S) - + def test_collect_0(self, debuglog): + self.gc.collect(1) # start a major + debuglog.reset() + self.gc.collect(0) # do ONLY a minor + assert debuglog.summary() == {'gc-minor': 1} + + def test_enable_disable(self, debuglog): + def large_malloc(): + # malloc an object which is large enough to trigger a major collection + threshold = self.gc.next_major_collection_threshold + self.malloc(VAR, int(threshold/8)) + summary = debuglog.summary() + debuglog.reset() + return summary + # + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + # + self.gc.disable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-minor'] + # + self.gc.enable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + + def test_call_collect_when_disabled(self, debuglog): + # malloc an object and put it the old generation + s = self.malloc(S) + s.x = 42 + self.stackroots.append(s) + self.gc.collect() + s = self.stackroots.pop() + # + self.gc.disable() + self.gc.collect(1) # start a major collect + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect(1) # run one more step + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect() # finish the major collection + summary = debuglog.summary() + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + # s is freed + py.test.raises(RuntimeError, 's.x') + + def test_collect_step(self, debuglog): + from rpython.rlib import rgc + n = 0 + states = [] + while True: + debuglog.reset() + val = self.gc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + summary = debuglog.summary() + assert summary == {'gc-minor': 1, 'gc-collect-step': 1} + if rgc.is_done(val): + break + n += 1 + if n == 100: + assert False, 'this looks like an endless loop' + # + assert states == [ + (incminimark.STATE_SCANNING, incminimark.STATE_MARKING), + (incminimark.STATE_MARKING, incminimark.STATE_SWEEPING), + (incminimark.STATE_SWEEPING, incminimark.STATE_FINALIZING), + (incminimark.STATE_FINALIZING, incminimark.STATE_SCANNING) + ] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -309,6 +309,12 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) + self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc], + annmodel.SomeInteger()) + self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None) + self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None) + self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc], + annmodel.s_Bool) self.can_move_ptr = getfn(GCClass.can_move.im_func, [s_gc, SomeAddress()], annmodel.SomeBool()) @@ -884,6 +890,28 @@ resultvar=op.result) self.pop_roots(hop, livevars) + def gct_gc__collect_step(self, hop): + op = hop.spaceop + livevars = self.push_roots(hop) + hop.genop("direct_call", [self.collect_step_ptr, self.c_const_gc], + resultvar=op.result) + self.pop_roots(hop, livevars) + + def gct_gc__enable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.enable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__disable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.disable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__isenabled(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.isenabled_ptr, self.c_const_gc], + resultvar=op.result) + def gct_gc_can_move(self, hop): op = hop.spaceop v_addr = hop.genop('cast_ptr_to_adr', diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,5 +1,6 @@ import sys import time +from collections import Counter from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry @@ -38,6 +39,23 @@ assert False, ("nesting error: no start corresponding to stop %r" % (category,)) + def reset(self): + # only for tests: empty the log + self[:] = [] + + def summary(self, flatten=False): + res = Counter() + def visit(lst): + for section, sublist in lst: + if section == 'debug_print': + continue + res[section] += 1 + if flatten: + visit(sublist) + # + visit(self) + return res + def __repr__(self): import pprint return pprint.pformat(list(self)) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -13,6 +13,50 @@ # General GC features collect = gc.collect +enable = gc.enable +disable = gc.disable +isenabled = gc.isenabled + +def collect_step(): + """ + If the GC is incremental, run a single gc-collect-step. + + Return an integer which encodes the starting and ending GC state. Use + rgc.{old_state,new_state,is_done} to decode it. + + If the GC is not incremental, do a full collection and return a value on + which rgc.is_done() return True. + """ + gc.collect() + return _encode_states(1, 0) + +def _encode_states(oldstate, newstate): + return oldstate << 8 | newstate + +def old_state(states): + return (states & 0xFF00) >> 8 + +def new_state(states): + return states & 0xFF + +def is_done(states): + """ + Return True if the return value of collect_step signals the end of a major + collection + """ + old = old_state(states) + new = new_state(states) + return is_done__states(old, new) + +def is_done__states(oldstate, newstate): + "Like is_done, but takes oldstate and newstate explicitly" + # a collection is considered done when it ends up in the starting state + # (which is usually represented as 0). This logic works for incminimark, + # which is currently the only gc actually used and for which collect_step + # is implemented. In case we add more GC in the future, we might want to + # delegate this logic to the GC itself, but for now it is MUCH simpler to + # just write it in plain RPython. + return oldstate != 0 and newstate == 0 def set_max_heap_size(nbytes): """Limit the heap size to n bytes. @@ -131,6 +175,44 @@ args_v = hop.inputargs(lltype.Signed) return hop.genop('gc__collect', args_v, resulttype=hop.r_result) + +class EnableDisableEntry(ExtRegistryEntry): + _about_ = (gc.enable, gc.disable) + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + hop.exception_cannot_occur() + opname = self.instance.__name__ + return hop.genop('gc__%s' % opname, hop.args_v, resulttype=hop.r_result) + + +class IsEnabledEntry(ExtRegistryEntry): + _about_ = gc.isenabled + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_Bool + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result) + + +class CollectStepEntry(ExtRegistryEntry): + _about_ = collect_step + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.SomeInteger() + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__collect_step', hop.args_v, resulttype=hop.r_result) + + class SetMaxHeapSizeEntry(ExtRegistryEntry): _about_ = set_max_heap_size diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py --- a/rpython/rlib/test/test_debug.py +++ b/rpython/rlib/test/test_debug.py @@ -1,5 +1,5 @@ - import py +import pytest from rpython.rlib.debug import (check_annotation, make_sure_not_resized, debug_print, debug_start, debug_stop, have_debug_prints, debug_offset, debug_flush, @@ -10,6 +10,12 @@ from rpython.rlib import debug from rpython.rtyper.test.test_llinterp import interpret, gengraph + at pytest.fixture +def debuglog(monkeypatch): + dlog = debug.DebugLog() + monkeypatch.setattr(debug, '_log', dlog) + return dlog + def test_check_annotation(): class Error(Exception): pass @@ -94,7 +100,7 @@ py.test.raises(NotAListOfChars, "interpret(g, [3])") -def test_debug_print_start_stop(): +def test_debug_print_start_stop(debuglog): def f(x): debug_start("mycat") debug_print("foo", 2, "bar", x) @@ -103,22 +109,27 @@ debug_offset() # should not explode at least return have_debug_prints() - try: - debug._log = dlog = debug.DebugLog() - res = f(3) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = f(3) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + debuglog.reset() - try: - debug._log = dlog = debug.DebugLog() - res = interpret(f, [3]) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = interpret(f, [3]) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] +def test_debuglog_summary(debuglog): + debug_start('foo') + debug_start('bar') # this is nested, so not counted in the summary by default + debug_stop('bar') + debug_stop('foo') + debug_start('foo') + debug_stop('foo') + debug_start('bar') + debug_stop('bar') + # + assert debuglog.summary() == {'foo': 2, 'bar': 1} + assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2} def test_debug_start_stop_timestamp(): import time diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -39,6 +39,45 @@ assert res is None +def test_enable_disable(): + def f(): + gc.enable() + a = gc.isenabled() + gc.disable() + b = gc.isenabled() + return a and not b + + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__enable', 'gc__isenabled', + 'gc__disable', 'gc__isenabled'] + res = interpret(f, []) + assert res + +def test_collect_step(): + def f(): + return rgc.collect_step() + + assert f() + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__collect_step'] + res = interpret(f, []) + assert res + +def test__encode_states(): + val = rgc._encode_states(42, 43) + assert rgc.old_state(val) == 42 + assert rgc.new_state(val) == 43 + assert not rgc.is_done(val) + # + val = rgc.collect_step() + assert rgc.is_done(val) + def test_can_move(): T0 = lltype.GcStruct('T') T1 = lltype.GcArray(lltype.Float) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -819,6 +819,18 @@ def op_gc__collect(self, *gen): self.heap.collect(*gen) + def op_gc__collect_step(self): + return self.heap.collect_step() + + def op_gc__enable(self): + self.heap.enable() + + def op_gc__disable(self): + self.heap.disable() + + def op_gc__isenabled(self): + return self.heap.isenabled() + def op_gc_heap_stats(self): raise NotImplementedError diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py --- a/rpython/rtyper/lltypesystem/llheap.py +++ b/rpython/rtyper/lltypesystem/llheap.py @@ -5,7 +5,7 @@ setfield = setattr from operator import setitem as setarrayitem -from rpython.rlib.rgc import can_move, collect, add_memory_pressure +from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure, collect_step def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue, offsets=None): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -456,6 +456,10 @@ # __________ GC operations __________ 'gc__collect': LLOp(canmallocgc=True), + 'gc__collect_step': LLOp(canmallocgc=True), + 'gc__enable': LLOp(), + 'gc__disable': LLOp(), + 'gc__isenabled': LLOp(), 'gc_free': LLOp(), 'gc_fetch_exception': LLOp(), 'gc_restore_exception': LLOp(), diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,6 +1812,92 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_enable_disable(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(should_disable): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + if should_disable: + gc.disable() + assert not gc.isenabled() + # try to trigger a major collection + N = 100 # this should be enough, increase if not + lst = [] + for i in range(N): + lst.append(chr(i%256) * (1024*1024)) + #print i, counter.val + # + gc.enable() + assert gc.isenabled() + return counter.val + return f + + def test_enable_disable(self): + # first, run with normal gc. If the assert fails it means that in the + # loop we don't allocate enough mem to trigger a major collection. Try + # to increase N + deleted = self.run("enable_disable", 0) + assert deleted == 1, 'This should not fail, try to increment N' + # + # now, run with gc.disable: this should NOT free x1 + deleted = self.run("enable_disable", 1) + assert deleted == 0 + + def define_collect_step(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + gc.disable() + n = 0 + states = [] + while True: + n += 1 + val = rgc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + if rgc.is_done(val): + break + if n == 100: + print 'Endless loop!' + assert False, 'this looks like an endless loop' + + if n < 4: # we expect at least 4 steps + print 'Too few steps! n =', n + assert False + + # check that the state transitions are reasonable + first_state, _ = states[0] + for i, (old_state, new_state) in enumerate(states): + is_last = (i == len(states) - 1) + is_valid = False + if is_last: + assert old_state != new_state == first_state + else: + assert new_state == old_state or new_state == old_state+1 + + return counter.val + return f + + def test_collect_step(self): + deleted = self.run("collect_step") + assert deleted == 1 + def define_total_gc_time(cls): def f(): l = [] diff --git a/rpython/translator/goal/gcbench.py b/rpython/translator/goal/gcbench.py --- a/rpython/translator/goal/gcbench.py +++ b/rpython/translator/goal/gcbench.py @@ -44,8 +44,9 @@ # - Results are sensitive to locking cost, but we dont # check for proper locking import time +import gc -USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N]""" +USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N] [--gc=off|--gc=manual]""" ENABLE_THREADS = True @@ -173,6 +174,7 @@ depths = DEFAULT_DEPTHS threads = 0 repeatcount = 1 + gc_policy = 'on' for arg in argv[1:]: if arg.startswith('--threads='): arg = arg[len('--threads='):] @@ -189,13 +191,22 @@ depths = [int(s) for s in arg] except ValueError: return argerror() + elif arg.startswith('--gc=off'): + gc_policy = 'off' + elif arg.startswith('--gc=manual'): + gc_policy = 'manual' else: try: repeatcount = int(arg) except ValueError: return argerror() + # + if gc_policy == 'off' or gc_policy == 'manual': + gc.disable() for i in range(repeatcount): main(depths, threads) + if gc_policy == 'manual': + gc.collect(1) return 0 From pypy.commits at gmail.com Thu Dec 20 06:31:22 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Dec 2018 03:31:22 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fast path for unicode, bytes Message-ID: <5c1b7d8a.1c69fb81.c211d.97e9@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95515:3b46fe8d4d44 Date: 2018-12-16 22:24 +0200 http://bitbucket.org/pypy/pypy/changeset/3b46fe8d4d44/ Log: fast path for unicode, bytes diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1111,7 +1111,11 @@ unicodehelper.check_ascii_or_raise(space, s) return space.newutf8(s, len(s)) if encoding == 'utf-8' or encoding == 'utf8': - s = space.charbuf_w(w_obj) + if (space.isinstance_w(w_obj, space.w_unicode) or + space.isinstance_w(w_obj, space.w_bytes)): + s = space.utf8_w(w_obj) + else: + s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) w_codecs = space.getbuiltinmodule("_codecs") From pypy.commits at gmail.com Thu Dec 20 06:31:24 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Dec 2018 03:31:24 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fast path for unicode, also raises on non-ascii strings Message-ID: <5c1b7d8c.1c69fb81.16261.7f23@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95516:72d83a806c67 Date: 2018-12-20 13:25 +0200 http://bitbucket.org/pypy/pypy/changeset/72d83a806c67/ Log: fast path for unicode, also raises on non-ascii strings diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1671,6 +1671,8 @@ # needed because CPython has the same issue. (Well, it's # unclear if there is any use at all for getting the bytes in # the unicode buffer.) + if self.isinstance_w(w_obj, self.w_unicode): + return w_obj.charbuf_w(self) try: return self.bytes_w(w_obj) except OperationError as e: From pypy.commits at gmail.com Thu Dec 20 13:36:08 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Dec 2018 10:36:08 -0800 (PST) Subject: [pypy-commit] pypy py3.6: solve 32bit _blake2 compilation on linux, still need win32, arm fixes Message-ID: <5c1be118.1c69fb81.30bf9.2b1c@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95517:e7b2b377385e Date: 2018-12-20 20:34 +0200 http://bitbucket.org/pypy/pypy/changeset/e7b2b377385e/ Log: solve 32bit _blake2 compilation on linux, still need win32, arm fixes diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -1,8 +1,18 @@ import os import sys +import platform from cffi import FFI +IS_ARM = platform.machine().startswith('arm') +if IS_ARM: + # XXX Choose neon accelaration + define_macros = [] +else: + define_macros = [('__SSE2__', '1')] + + + blake_cdef = """ #define BLAKE_OUTBYTES ... #define BLAKE_SALTBYTES ... @@ -72,6 +82,7 @@ sources=[os.path.join(_libdir, 'blake2b.c'), ], include_dirs=[_libdir], + define_macros=define_macros, ) def _replace_b2s(src): @@ -87,6 +98,7 @@ sources=[os.path.join(_libdir, 'blake2s.c'), ], include_dirs=[_libdir], + define_macros=define_macros, ) if __name__ == '__main__': From pypy.commits at gmail.com Fri Dec 21 05:58:11 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 21 Dec 2018 02:58:11 -0800 (PST) Subject: [pypy-commit] pypy default: Py_ssize_t is fixed as a long, c++ makes the format warning an error Message-ID: <5c1cc743.1c69fb81.7eab4.b8cf@mx.google.com> Author: Matti Picus Branch: Changeset: r95518:b11298a062cb Date: 2018-12-21 12:55 +0200 http://bitbucket.org/pypy/pypy/changeset/b11298a062cb/ Log: Py_ssize_t is fixed as a long, c++ makes the format warning an error diff --git a/pypy/module/cpyext/src/stringobject.c b/pypy/module/cpyext/src/stringobject.c --- a/pypy/module/cpyext/src/stringobject.c +++ b/pypy/module/cpyext/src/stringobject.c @@ -164,7 +164,7 @@ va_arg(vargs, PY_LONG_LONG)); #endif else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "d", + sprintf(s, "%ld", va_arg(vargs, Py_ssize_t)); else sprintf(s, "%d", va_arg(vargs, int)); From pypy.commits at gmail.com Fri Dec 21 05:58:13 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 21 Dec 2018 02:58:13 -0800 (PST) Subject: [pypy-commit] pypy default: do not force 64bit architecture Message-ID: <5c1cc745.1c69fb81.98724.bd67@mx.google.com> Author: Matti Picus Branch: Changeset: r95519:0d8bf1346514 Date: 2018-12-21 12:56 +0200 http://bitbucket.org/pypy/pypy/changeset/0d8bf1346514/ Log: do not force 64bit architecture diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -15,7 +15,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) @@ -25,7 +25,7 @@ PLATFORM := $(shell uname -s) ifeq ($(PLATFORM),Darwin) - cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup + cppflags+=-dynamiclib -single_module -undefined dynamic_lookup endif From pypy.commits at gmail.com Sun Dec 23 05:30:39 2018 From: pypy.commits at gmail.com (antocuni) Date: Sun, 23 Dec 2018 02:30:39 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: add a blog post about the gc-disable branch Message-ID: <5c1f63cf.1c69fb81.47bda.b20f@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5923:3c9e148d058f Date: 2018-12-23 11:29 +0100 http://bitbucket.org/pypy/extradoc/changeset/3c9e148d058f/ Log: add a blog post about the gc-disable branch diff --git a/blog/draft/2018-12-gc-disable.rst b/blog/draft/2018-12-gc-disable.rst new file mode 100644 --- /dev/null +++ b/blog/draft/2018-12-gc-disable.rst @@ -0,0 +1,102 @@ +PyPy for low-latency systems +============================= + +Recently I have merged the gc-disable branch, introducing a couple of features +which are useful when you need to respond to certain events with the lowest +possible latency. This work has been kindly sponsored by `Gambit Research`_ +(which, by the way, is a very cool and geeky place where to work, in case you +are interested_). + +The PyPy VM manages the memory using a generational, moving Garbage Collector: +periodically, the GC scans the whole heap to find unreachable objects and +frees the corresponding memory. Although at a first look this strategy might +sound expensive, in practice the total cost of memory management is far less +than e.g. on CPython, which is based on reference counting. This happens for +various reasons, the most important ones being that allocation is very fast +(especially compared to malloc-based allocators), and deallocation of objects +which die young is basically for free. More information about the PyPy GC is +available here_. + +As we said, the total cost of memory managment is less on PyPy than on +CPython, and it's one of the reasons why PyPy is so fast. However, one big +disadvantage is that while on CPython the cost of memory management is spread +all over the execution of the program, on PyPy it is concentrated when the GC +runs, causing observable pauses which interrupt the execution of the user +program. + +To avoid excessively long pauses, the PyPy GC has been using an `incremental +strategy since 2013`_: the GC runs as a series of "steps", letting the user +program to progress between each step. + +The following chart shows the behavior of a real-world, long-running process: + +.. image:: 2018-12-gc-timing.png + +The orange line shows the amount of memory used by the program, which +increases linearly while the program progresses. Every ~5 minutes, the GC +kicks in and the memory usage drops from ~5.2GB to ~2.8GB (this is not a +casual ratio, it is controlled by the PYPY_GC_MAJOR_COLLECT_ env variable). + +The purple line shows aggregated data about the GC timing: the whole +collection takes ~1400 individual steps over the course of ~1 minute: each +point represent the **maximum** time a single step took during the past 10 +seconds. Most steps take ~10-20 ms, although we see a horrible peak of ~100 ms +towards the end. We have not investigated yet what it is caused by, but we +suspect it is related with the deallocation of raw objects. + +This is clearly a problem for systems where it is important to respond to +certain events with a latency which is both low and consistent: the GC kicks +in at the wrong time, it might causes unacceptable pauses during the response. + +Let's look again at our real-world example: this is a system which +continuously monitors an external stream; when a certain event occurs, we want +to take an action. The following chart shows the maximum time it takes to +complete one of such actions, aggregated every minute: + +.. image:: 2018-12-normal-max.png + +You can clearly see that the baseline response time is around ~20-30 +ms. However, we can also see periodic spikes around ~50-100 ms, with peaks up +to ~350-450 ms! After a bit of investigation, we concluded that most (although +not all) of the spikes were caused by the GC kicking in at the wrong time. + +The work I did in the ``gc-disable`` branch aims to fix this problem by +introducing `two new features`_ to the ``gc`` module: + + - ``gc.disable()``, which used to simply inhibits the execution of + finalizers, now disables the GC major collections for real. After a call + to it, you will see the memory usage to grow indefinitely. + + - ``gc.collect_step()`` is a new function which you can use to manually + execute a single incremental step. + +Combining these two functions, it is possible to take control of the GC to +make sure it runs only when it is acceptable to do so. For an example of +usage, you can look at the implementation of a `custom GC`_ inside pypytools_. +The peculiarity is that is also defines a ``with nogc():`` context manager +which you can use to mark performance-critical sections where the GC is not +allowed to run. + +The following chart compares the behavior of the default PyPy GC and the new +custom GC, after a careful placing of ``nogc()`` sections: + +.. image:: 2018-12-nogc.png + +The yellow line is the same as before, while the purple line shows the new +system: almost all spikes have gone, and the baseline performance is about 10% +better. There is still one spike towards the end, but after some investigation +we concluded that it was **not** caused by the GC. + +All in all, a pretty big success, I think. These functionalities are already +available in the nightly builds of PyPy, and will be included in the next +release: take this as a Christmas present :) + + +.. _`Gambit Research`: https://www.gambitresearch.com/ +.. _interested: https://www.gambitresearch.com/jobs.html +.. _here: https://pypy.readthedocs.io/en/latest/gc_info.html#incminimark +.. _`incremental strategy since 2013`: https://morepypy.blogspot.com/2013/10/incremental-garbage-collector-in-pypy.html +.. _PYPY_GC_MAJOR_COLLECT: https://pypy.readthedocs.io/en/latest/gc_info.html#environment-variables +.. _`two new features`: https://pypy.readthedocs.io/en/latest/gc_info.html#semi-manual-gc-management +.. _`Custom GC`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default +.. _pypytools: https://pypi.org/project/pypytools/ diff --git a/blog/draft/2018-12-gc-timing.png b/blog/draft/2018-12-gc-timing.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cfa8335b6d21b94e36a44f930c3bd01cb2affa3e GIT binary patch [cut] diff --git a/blog/draft/2018-12-nogc-max.png b/blog/draft/2018-12-nogc-max.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ae9d5ad296f82a8d56408c2818691dbefa0fd301 GIT binary patch [cut] diff --git a/blog/draft/2018-12-normal-max.png b/blog/draft/2018-12-normal-max.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..29b3381a8162378ae89e1eeeaae9ae8b5cf3d838 GIT binary patch [cut] From pypy.commits at gmail.com Sun Dec 23 05:36:15 2018 From: pypy.commits at gmail.com (antocuni) Date: Sun, 23 Dec 2018 02:36:15 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: move the blog post and related images in its own directory Message-ID: <5c1f651f.1c69fb81.9d9a6.a508@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5924:b60c0f4fb0b5 Date: 2018-12-23 11:35 +0100 http://bitbucket.org/pypy/extradoc/changeset/b60c0f4fb0b5/ Log: move the blog post and related images in its own directory diff --git a/blog/draft/2018-12-gc-disable.rst b/blog/draft/2018-12-gc-disable/gc-disable.rst rename from blog/draft/2018-12-gc-disable.rst rename to blog/draft/2018-12-gc-disable/gc-disable.rst --- a/blog/draft/2018-12-gc-disable.rst +++ b/blog/draft/2018-12-gc-disable/gc-disable.rst @@ -30,7 +30,7 @@ The following chart shows the behavior of a real-world, long-running process: -.. image:: 2018-12-gc-timing.png +.. image:: gc-timing.png The orange line shows the amount of memory used by the program, which increases linearly while the program progresses. Every ~5 minutes, the GC @@ -53,7 +53,7 @@ to take an action. The following chart shows the maximum time it takes to complete one of such actions, aggregated every minute: -.. image:: 2018-12-normal-max.png +.. image:: normal-max.png You can clearly see that the baseline response time is around ~20-30 ms. However, we can also see periodic spikes around ~50-100 ms, with peaks up @@ -80,7 +80,7 @@ The following chart compares the behavior of the default PyPy GC and the new custom GC, after a careful placing of ``nogc()`` sections: -.. image:: 2018-12-nogc.png +.. image:: nogc-max.png The yellow line is the same as before, while the purple line shows the new system: almost all spikes have gone, and the baseline performance is about 10% diff --git a/blog/draft/2018-12-gc-timing.png b/blog/draft/2018-12-gc-disable/gc-timing.png rename from blog/draft/2018-12-gc-timing.png rename to blog/draft/2018-12-gc-disable/gc-timing.png diff --git a/blog/draft/2018-12-nogc-max.png b/blog/draft/2018-12-gc-disable/nogc-max.png rename from blog/draft/2018-12-nogc-max.png rename to blog/draft/2018-12-gc-disable/nogc-max.png diff --git a/blog/draft/2018-12-normal-max.png b/blog/draft/2018-12-gc-disable/normal-max.png rename from blog/draft/2018-12-normal-max.png rename to blog/draft/2018-12-gc-disable/normal-max.png From pypy.commits at gmail.com Sun Dec 23 05:44:14 2018 From: pypy.commits at gmail.com (antocuni) Date: Sun, 23 Dec 2018 02:44:14 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: fix link and fix a typo Message-ID: <5c1f66fe.1c69fb81.fdee4.6bcf@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5925:62f53d229f99 Date: 2018-12-23 11:43 +0100 http://bitbucket.org/pypy/extradoc/changeset/62f53d229f99/ Log: fix link and fix a typo diff --git a/blog/draft/2018-12-gc-disable/gc-disable.rst b/blog/draft/2018-12-gc-disable/gc-disable.rst --- a/blog/draft/2018-12-gc-disable/gc-disable.rst +++ b/blog/draft/2018-12-gc-disable/gc-disable.rst @@ -4,8 +4,8 @@ Recently I have merged the gc-disable branch, introducing a couple of features which are useful when you need to respond to certain events with the lowest possible latency. This work has been kindly sponsored by `Gambit Research`_ -(which, by the way, is a very cool and geeky place where to work, in case you -are interested_). +(which, by the way, is a very cool and geeky place where to work_, in case you +are interested). The PyPy VM manages the memory using a generational, moving Garbage Collector: periodically, the GC scans the whole heap to find unreachable objects and @@ -63,7 +63,7 @@ The work I did in the ``gc-disable`` branch aims to fix this problem by introducing `two new features`_ to the ``gc`` module: - - ``gc.disable()``, which used to simply inhibits the execution of + - ``gc.disable()``, which used to simply inhibit the execution of finalizers, now disables the GC major collections for real. After a call to it, you will see the memory usage to grow indefinitely. @@ -93,7 +93,7 @@ .. _`Gambit Research`: https://www.gambitresearch.com/ -.. _interested: https://www.gambitresearch.com/jobs.html +.. _work: https://www.gambitresearch.com/jobs.html .. _here: https://pypy.readthedocs.io/en/latest/gc_info.html#incminimark .. _`incremental strategy since 2013`: https://morepypy.blogspot.com/2013/10/incremental-garbage-collector-in-pypy.html .. _PYPY_GC_MAJOR_COLLECT: https://pypy.readthedocs.io/en/latest/gc_info.html#environment-variables From pypy.commits at gmail.com Sun Dec 23 06:08:56 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 23 Dec 2018 03:08:56 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: edit the blog post Message-ID: <5c1f6cc8.1c69fb81.d8910.f97c@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5926:b52d3e5d3a0d Date: 2018-12-23 13:08 +0200 http://bitbucket.org/pypy/extradoc/changeset/b52d3e5d3a0d/ Log: edit the blog post diff --git a/blog/draft/2018-12-gc-disable/gc-disable.rst b/blog/draft/2018-12-gc-disable/gc-disable.rst --- a/blog/draft/2018-12-gc-disable/gc-disable.rst +++ b/blog/draft/2018-12-gc-disable/gc-disable.rst @@ -7,48 +7,49 @@ (which, by the way, is a very cool and geeky place where to work_, in case you are interested). -The PyPy VM manages the memory using a generational, moving Garbage Collector: -periodically, the GC scans the whole heap to find unreachable objects and +The PyPy VM manages memory using a generational, moving Garbage Collector. +Periodically, the GC scans the whole heap to find unreachable objects and frees the corresponding memory. Although at a first look this strategy might sound expensive, in practice the total cost of memory management is far less -than e.g. on CPython, which is based on reference counting. This happens for -various reasons, the most important ones being that allocation is very fast -(especially compared to malloc-based allocators), and deallocation of objects -which die young is basically for free. More information about the PyPy GC is -available here_. +than e.g. on CPython, which is based on reference counting. While maybe +counter-intuitive, the main advantage of a non-refcount strategy is +that allocation is very fast (especially compared to malloc-based allocators), +and deallocation of objects which die young is basically for free. More +information about the PyPy GC is available here_. As we said, the total cost of memory managment is less on PyPy than on CPython, and it's one of the reasons why PyPy is so fast. However, one big disadvantage is that while on CPython the cost of memory management is spread -all over the execution of the program, on PyPy it is concentrated when the GC +all over the execution of the program, on PyPy it is concentrated into GC runs, causing observable pauses which interrupt the execution of the user program. To avoid excessively long pauses, the PyPy GC has been using an `incremental -strategy since 2013`_: the GC runs as a series of "steps", letting the user +strategy`_ since 2013. The GC runs as a series of "steps", letting the user program to progress between each step. The following chart shows the behavior of a real-world, long-running process: .. image:: gc-timing.png -The orange line shows the amount of memory used by the program, which +The orange line shows the total memory used by the program, which increases linearly while the program progresses. Every ~5 minutes, the GC -kicks in and the memory usage drops from ~5.2GB to ~2.8GB (this is not a -casual ratio, it is controlled by the PYPY_GC_MAJOR_COLLECT_ env variable). +kicks in and the memory usage drops from ~5.2GB to ~2.8GB (this is controlled +by the PYPY_GC_MAJOR_COLLECT_ env variable). The purple line shows aggregated data about the GC timing: the whole collection takes ~1400 individual steps over the course of ~1 minute: each point represent the **maximum** time a single step took during the past 10 seconds. Most steps take ~10-20 ms, although we see a horrible peak of ~100 ms towards the end. We have not investigated yet what it is caused by, but we -suspect it is related with the deallocation of raw objects. +suspect it is related to the deallocation of raw objects. -This is clearly a problem for systems where it is important to respond to -certain events with a latency which is both low and consistent: the GC kicks -in at the wrong time, it might causes unacceptable pauses during the response. +These multi-millesecond pauses are a problem for systems where it is important +to respond to certain events with a latency which is both low and consistent. +The GC kicks in at the wrong time, it might causes unacceptable pauses during +the collection cycle. -Let's look again at our real-world example: this is a system which +Let's look again at our real-world example. This is a system which continuously monitors an external stream; when a certain event occurs, we want to take an action. The following chart shows the maximum time it takes to complete one of such actions, aggregated every minute: @@ -63,12 +64,13 @@ The work I did in the ``gc-disable`` branch aims to fix this problem by introducing `two new features`_ to the ``gc`` module: - - ``gc.disable()``, which used to simply inhibit the execution of - finalizers, now disables the GC major collections for real. After a call - to it, you will see the memory usage to grow indefinitely. + - ``gc.disable()``, which previously only inhibited the execution of + finalizers without actually touching the GC, now disables the GC major + collections. After a call to it, you will see the memory usage grow + indefinitely. - ``gc.collect_step()`` is a new function which you can use to manually - execute a single incremental step. + execute a single incremental GC collection step. Combining these two functions, it is possible to take control of the GC to make sure it runs only when it is acceptable to do so. For an example of @@ -91,11 +93,13 @@ available in the nightly builds of PyPy, and will be included in the next release: take this as a Christmas present :) +Antonio Cuni and the PyPy team + .. _`Gambit Research`: https://www.gambitresearch.com/ .. _work: https://www.gambitresearch.com/jobs.html .. _here: https://pypy.readthedocs.io/en/latest/gc_info.html#incminimark -.. _`incremental strategy since 2013`: https://morepypy.blogspot.com/2013/10/incremental-garbage-collector-in-pypy.html +.. _`incremental strategy`: https://morepypy.blogspot.com/2013/10/incremental-garbage-collector-in-pypy.html .. _PYPY_GC_MAJOR_COLLECT: https://pypy.readthedocs.io/en/latest/gc_info.html#environment-variables .. _`two new features`: https://pypy.readthedocs.io/en/latest/gc_info.html#semi-manual-gc-management .. _`Custom GC`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default From pypy.commits at gmail.com Sun Dec 23 08:51:52 2018 From: pypy.commits at gmail.com (antocuni) Date: Sun, 23 Dec 2018 05:51:52 -0800 (PST) Subject: [pypy-commit] =?utf-8?q?extradoc_extradoc=3A_typo=2C_thanks_to_R?= =?utf-8?q?en=C3=A9_Dudfield?= Message-ID: <5c1f92f8.1c69fb81.d0ec7.2e25@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5927:85b946c1b8aa Date: 2018-12-23 14:45 +0100 http://bitbucket.org/pypy/extradoc/changeset/85b946c1b8aa/ Log: typo, thanks to René Dudfield diff --git a/blog/draft/2018-12-gc-disable/gc-disable.rst b/blog/draft/2018-12-gc-disable/gc-disable.rst --- a/blog/draft/2018-12-gc-disable/gc-disable.rst +++ b/blog/draft/2018-12-gc-disable/gc-disable.rst @@ -75,7 +75,7 @@ Combining these two functions, it is possible to take control of the GC to make sure it runs only when it is acceptable to do so. For an example of usage, you can look at the implementation of a `custom GC`_ inside pypytools_. -The peculiarity is that is also defines a ``with nogc():`` context manager +The peculiarity is that it also defines a ``with nogc():`` context manager which you can use to mark performance-critical sections where the GC is not allowed to run. From pypy.commits at gmail.com Sun Dec 23 08:51:54 2018 From: pypy.commits at gmail.com (antocuni) Date: Sun, 23 Dec 2018 05:51:54 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: typo Message-ID: <5c1f92fa.1c69fb81.5b5ff.1487@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5928:303c2decb9fc Date: 2018-12-23 14:51 +0100 http://bitbucket.org/pypy/extradoc/changeset/303c2decb9fc/ Log: typo diff --git a/blog/draft/2018-12-gc-disable/gc-disable.rst b/blog/draft/2018-12-gc-disable/gc-disable.rst --- a/blog/draft/2018-12-gc-disable/gc-disable.rst +++ b/blog/draft/2018-12-gc-disable/gc-disable.rst @@ -46,7 +46,7 @@ These multi-millesecond pauses are a problem for systems where it is important to respond to certain events with a latency which is both low and consistent. -The GC kicks in at the wrong time, it might causes unacceptable pauses during +If the GC kicks in at the wrong time, it might causes unacceptable pauses during the collection cycle. Let's look again at our real-world example. This is a system which From pypy.commits at gmail.com Sun Dec 23 09:57:14 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 23 Dec 2018 06:57:14 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5c1fa24a.1c69fb81.97f39.098b@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95520:406030262749 Date: 2018-12-23 16:53 +0200 http://bitbucket.org/pypy/pypy/changeset/406030262749/ Log: merge default into branch diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -327,6 +327,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(self): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py --- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py +++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py @@ -244,6 +244,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(): ffi = _cffi1_backend.FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -1654,6 +1654,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_all_primitives(self): assert set(PRIMITIVE_TO_INDEX) == set([ diff --git a/extra_tests/test_pyrepl/conftest.py b/extra_tests/test_pyrepl/conftest.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/conftest.py @@ -0,0 +1,8 @@ +import sys + +def pytest_ignore_collect(path): + if '__pypy__' not in sys.builtin_module_names: + try: + import pyrepl + except ImportError: + return True diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -341,7 +341,7 @@ # """ # note that 'buffer' is a type, set on this instance by __init__ - def from_buffer(self, python_buffer): + def from_buffer(self, python_buffer, require_writable=False): """Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is not meant to be used on the built-in types @@ -349,7 +349,8 @@ but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ - return self._backend.from_buffer(self.BCharA, python_buffer) + return self._backend.from_buffer(self.BCharA, python_buffer, + require_writable) def memmove(self, dest, src, n): """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -401,8 +401,10 @@ * some functions and attributes of the ``gc`` module behave in a slightly different way: for example, ``gc.enable`` and - ``gc.disable`` are supported, but instead of enabling and disabling - the GC, they just enable and disable the execution of finalizers. + ``gc.disable`` are supported, but "enabling and disabling the GC" has + a different meaning in PyPy than in CPython. These functions + actually enable and disable the major collections and the + execution of finalizers. * PyPy prints a random line from past #pypy IRC topics at startup in interactive mode. In a released version, this behaviour is suppressed, but diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -22,8 +22,44 @@ larger. (A third category, the very large objects, are initially allocated outside the nursery and never move.) -Since Incminimark is an incremental GC, the major collection is incremental, -meaning there should not be any pauses longer than 1ms. +Since Incminimark is an incremental GC, the major collection is incremental: +the goal is not to have any pause longer than 1ms, but in practice it depends +on the size and characteristics of the heap: occasionally, there can be pauses +between 10-100ms. + + +Semi-manual GC management +-------------------------- + +If there are parts of the program where it is important to have a low latency, +you might want to control precisely when the GC runs, to avoid unexpected +pauses. Note that this has effect only on major collections, while minor +collections continue to work as usual. + +As explained above, a full major collection consists of ``N`` steps, where +``N`` depends on the size of the heap; generally speaking, it is not possible +to predict how many steps will be needed to complete a collection. + +``gc.enable()`` and ``gc.disable()`` control whether the GC runs collection +steps automatically. When the GC is disabled the memory usage will grow +indefinitely, unless you manually call ``gc.collect()`` and +``gc.collect_step()``. + +``gc.collect()`` runs a full major collection. + +``gc.collect_step()`` runs a single collection step. It returns an object of +type GcCollectStepStats_, the same which is passed to the corresponding `GC +Hooks`_. The following code is roughly equivalent to a ``gc.collect()``:: + + while True: + if gc.collect_step().major_is_done: + break + +For a real-world example of usage of this API, you can look at the 3rd-party +module `pypytools.gc.custom`_, which also provides a ``with customgc.nogc()`` +context manager to mark sections where the GC is forbidden. + +.. _`pypytools.gc.custom`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default Fragmentation @@ -184,6 +220,8 @@ the number of pinned objects. +.. _GcCollectStepStats: + The attributes for ``GcCollectStepStats`` are: ``count``, ``duration``, ``duration_min``, ``duration_max`` @@ -192,10 +230,14 @@ ``oldstate``, ``newstate`` Integers which indicate the state of the GC before and after the step. +``major_is_done`` + Boolean which indicate whether this was the last step of the major + collection + The value of ``oldstate`` and ``newstate`` is one of these constants, defined inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, -``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string -representation of it by indexing the ``GC_STATS`` tuple. +``STATE_SWEEPING``, ``STATE_FINALIZING``, ``STATE_USERDEL``. It is possible +to get a string representation of it by indexing the ``GC_STATES`` tuple. The attributes for ``GcCollectStats`` are: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -60,3 +60,10 @@ .. branch: cleanup-test_lib_pypy Update most test_lib_pypy/ tests and move them to extra_tests/. + +.. branch: gc-disable + +Make it possible to manually manage the GC by using a combination of +gc.disable() and gc.collect_step(). Make sure to write a proper release +announcement in which we explain that existing programs could leak memory if +they run for too much time between a gc.disable()/gc.enable() diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -327,7 +327,8 @@ return w_ctype.cast(w_ob) - def descr_from_buffer(self, w_python_buffer): + @unwrap_spec(require_writable=int) + def descr_from_buffer(self, w_python_buffer, require_writable=0): """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is @@ -337,7 +338,8 @@ 'array.array' or numpy arrays.""" # w_ctchara = newtype._new_chara_type(self.space) - return func._from_buffer(self.space, w_ctchara, w_python_buffer) + return func._from_buffer(self.space, w_ctchara, w_python_buffer, + require_writable) @unwrap_spec(w_arg=W_CData) diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -110,8 +110,8 @@ def _fetch_as_write_buffer(space, w_x): return space.writebuf_w(w_x) - at unwrap_spec(w_ctype=ctypeobj.W_CType) -def from_buffer(space, w_ctype, w_x): + at unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int) +def from_buffer(space, w_ctype, w_x, require_writable=0): from pypy.module._cffi_backend import ctypearray, ctypeprim # if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or @@ -119,13 +119,16 @@ raise oefmt(space.w_TypeError, "needs 'char[]', got '%s'", w_ctype.name) # - return _from_buffer(space, w_ctype, w_x) + return _from_buffer(space, w_ctype, w_x, require_writable) -def _from_buffer(space, w_ctype, w_x): +def _from_buffer(space, w_ctype, w_x, require_writable): if space.isinstance_w(w_x, space.w_unicode): raise oefmt(space.w_TypeError, - "from_buffer() cannot return the address a unicode") - buf = _fetch_as_read_buffer(space, w_x) + "from_buffer() cannot return the address of a unicode object") + if require_writable: + buf = _fetch_as_write_buffer(space, w_x) + else: + buf = _fetch_as_read_buffer(space, w_x) if space.isinstance_w(w_x, space.w_bytes): _cdata = get_raw_address_of_string(space, w_x) else: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3730,6 +3730,18 @@ check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") +def test_from_buffer_require_writable(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p1 = from_buffer(BCharA, b"foo", False) + assert p1 == from_buffer(BCharA, b"foo", False) + py.test.raises((TypeError, BufferError), from_buffer, BCharA, b"foo", True) + ba = bytearray(b"foo") + p1 = from_buffer(BCharA, ba, True) + p1[0] = b"g" + assert ba == b"goo" + def test_memmove(): Short = new_primitive_type("short") ShortA = new_array_type(new_pointer_type(Short), None) diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -287,6 +287,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_from_buffer_BytesIO(self): from _cffi_backend import FFI diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -15,7 +15,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) @@ -25,7 +25,7 @@ PLATFORM := $(shell uname -s) ifeq ($(PLATFORM),Darwin) - cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup + cppflags+=-dynamiclib -single_module -undefined dynamic_lookup endif diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -4,6 +4,7 @@ class Module(MixedModule): interpleveldefs = { 'collect': 'interp_gc.collect', + 'collect_step': 'interp_gc.collect_step', 'enable': 'interp_gc.enable', 'disable': 'interp_gc.disable', 'isenabled': 'interp_gc.isenabled', diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,5 +1,6 @@ from rpython.memory.gc.hook import GcHooks -from rpython.memory.gc import incminimark +from rpython.memory.gc import incminimark +from rpython.rlib import rgc from rpython.rlib.nonconst import NonConstant from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault @@ -117,12 +118,24 @@ self.descr_set_on_gc_collect(space, space.w_None) -class GcMinorHookAction(AsyncAction): +class NoRecursiveAction(AsyncAction): + depth = 0 + + def perform(self, ec, frame): + if self.depth == 0: + try: + self.depth += 1 + return self._do_perform(ec, frame) + finally: + self.depth -= 1 + + +class GcMinorHookAction(NoRecursiveAction): total_memory_used = 0 pinned_objects = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -145,7 +158,7 @@ self.pinned_objects = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcMinorStats( self.count, self.duration, @@ -157,12 +170,12 @@ self.space.call_function(self.w_callable, w_stats) -class GcCollectStepHookAction(AsyncAction): +class GcCollectStepHookAction(NoRecursiveAction): oldstate = 0 newstate = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -185,19 +198,20 @@ self.newstate = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStepStats( self.count, self.duration, self.duration_min, self.duration_max, self.oldstate, - self.newstate) + self.newstate, + rgc.is_done__states(self.oldstate, self.newstate)) self.reset() self.space.call_function(self.w_callable, w_stats) -class GcCollectHookAction(AsyncAction): +class GcCollectHookAction(NoRecursiveAction): num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -206,7 +220,7 @@ rawmalloc_bytes_after = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -227,7 +241,7 @@ self.rawmalloc_bytes_after = NonConstant(r_uint(42)) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStats(self.count, self.num_major_collects, self.arenas_count_before, @@ -252,15 +266,32 @@ class W_GcCollectStepStats(W_Root): + # NOTE: this is specific to incminimark: if we want to integrate the + # applevel gc module with another gc, we probably need a more general + # approach to this. + # + # incminimark has 4 GC states: scanning, marking, sweeping and + # finalizing. However, from the user point of view, we have an additional + # "virtual" state: USERDEL, which represent when we run applevel + # finalizers after having completed a GC major collection. This state is + # never explicitly visible when using hooks, but it is used for the return + # value of gc.collect_step (see interp_gc.py) + STATE_SCANNING = incminimark.STATE_SCANNING + STATE_MARKING = incminimark.STATE_MARKING + STATE_SWEEPING = incminimark.STATE_SWEEPING + STATE_FINALIZING = incminimark.STATE_FINALIZING + STATE_USERDEL = incminimark.STATE_FINALIZING + 1 # used by StepCollector + GC_STATES = tuple(incminimark.GC_STATES + ['USERDEL']) def __init__(self, count, duration, duration_min, duration_max, - oldstate, newstate): + oldstate, newstate, major_is_done): self.count = count self.duration = duration self.duration_min = duration_min self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate + self.major_is_done = major_is_done class W_GcCollectStats(W_Root): @@ -320,11 +351,16 @@ W_GcCollectStepStats.typedef = TypeDef( "GcCollectStepStats", - STATE_SCANNING = incminimark.STATE_SCANNING, - STATE_MARKING = incminimark.STATE_MARKING, - STATE_SWEEPING = incminimark.STATE_SWEEPING, - STATE_FINALIZING = incminimark.STATE_FINALIZING, - GC_STATES = tuple(incminimark.GC_STATES), + STATE_SCANNING = W_GcCollectStepStats.STATE_SCANNING, + STATE_MARKING = W_GcCollectStepStats.STATE_MARKING, + STATE_SWEEPING = W_GcCollectStepStats.STATE_SWEEPING, + STATE_FINALIZING = W_GcCollectStepStats.STATE_FINALIZING, + STATE_USERDEL = W_GcCollectStepStats.STATE_USERDEL, + GC_STATES = tuple(W_GcCollectStepStats.GC_STATES), + major_is_done = interp_attrproperty( + "major_is_done", + cls=W_GcCollectStepStats, + wrapfn="newbool"), **wrap_many(W_GcCollectStepStats, ( "count", "duration", diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import oefmt from rpython.rlib import rgc +from pypy.module.gc.hook import W_GcCollectStepStats @unwrap_spec(generation=int) @@ -16,7 +17,9 @@ cache.clear() rgc.collect() + _run_finalizers(space) +def _run_finalizers(space): # if we are running in gc.disable() mode but gc.collect() is called, # we should still call the finalizers now. We do this as an attempt # to get closer to CPython's behavior: in Py3.5 some tests @@ -39,18 +42,20 @@ return space.newint(0) def enable(space): - """Non-recursive version. Enable finalizers now. + """Non-recursive version. Enable major collections and finalizers. If they were already enabled, no-op. If they were disabled even several times, enable them anyway. """ + rgc.enable() if not space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = True enable_finalizers(space) def disable(space): - """Non-recursive version. Disable finalizers now. Several calls - to this function are ignored. + """Non-recursive version. Disable major collections and finalizers. + Multiple calls to this function are ignored. """ + rgc.disable() if space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = False disable_finalizers(space) @@ -77,6 +82,59 @@ if uda.pending_with_disabled_del is None: uda.pending_with_disabled_del = [] + +class StepCollector(object): + """ + Invoke rgc.collect_step() until we are done, then run the app-level + finalizers as a separate step + """ + + def __init__(self, space): + self.space = space + self.finalizing = False + + def do(self): + if self.finalizing: + self._run_finalizers() + self.finalizing = False + oldstate = W_GcCollectStepStats.STATE_USERDEL + newstate = W_GcCollectStepStats.STATE_SCANNING + major_is_done = True # now we are finally done + else: + states = self._collect_step() + oldstate = rgc.old_state(states) + newstate = rgc.new_state(states) + major_is_done = False # USERDEL still to do + if rgc.is_done(states): + newstate = W_GcCollectStepStats.STATE_USERDEL + self.finalizing = True + # + duration = -1 + return W_GcCollectStepStats( + count = 1, + duration = duration, + duration_min = duration, + duration_max = duration, + oldstate = oldstate, + newstate = newstate, + major_is_done = major_is_done) + + def _collect_step(self): + return rgc.collect_step() + + def _run_finalizers(self): + _run_finalizers(self.space) + +def collect_step(space): + """ + If the GC is incremental, run a single gc-collect-step. Return True when + the major collection is completed. + If the GC is not incremental, do a full collection and return True. + """ + sc = space.fromcache(StepCollector) + w_stats = sc.do() + return w_stats + # ____________________________________________________________ @unwrap_spec(filename='fsencode') diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -1,7 +1,20 @@ import py - +import pytest +from rpython.rlib import rgc +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.module.gc.interp_gc import StepCollector, W_GcCollectStepStats class AppTestGC(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + def rgc_isenabled(space): + return space.newbool(rgc.isenabled()) + cls.w_rgc_isenabled = space.wrap(interp2app(rgc_isenabled)) + def test_collect(self): import gc gc.collect() # mostly a "does not crash" kind of test @@ -63,12 +76,16 @@ def test_enable(self): import gc assert gc.isenabled() + assert self.rgc_isenabled() gc.disable() assert not gc.isenabled() + assert not self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() def test_gc_collect_overrides_gc_disable(self): import gc @@ -83,6 +100,24 @@ assert deleted == [1] gc.enable() + def test_gc_collect_step(self): + import gc + + class X(object): + deleted = 0 + def __del__(self): + X.deleted += 1 + + gc.disable() + X(); X(); X(); + n = 0 + while True: + n += 1 + if gc.collect_step().major_is_done: + break + + assert n >= 2 # at least one step + 1 finalizing + assert X.deleted == 3 class AppTestGcDumpHeap(object): pytestmark = py.test.mark.xfail(run=False) @@ -156,3 +191,55 @@ gc.collect() # the classes C should all go away here for r in rlist: assert r() is None + + +def test_StepCollector(): + W = W_GcCollectStepStats + SCANNING = W.STATE_SCANNING + MARKING = W.STATE_MARKING + SWEEPING = W.STATE_SWEEPING + FINALIZING = W.STATE_FINALIZING + USERDEL = W.STATE_USERDEL + + class MyStepCollector(StepCollector): + my_steps = 0 + my_done = False + my_finalized = 0 + + def __init__(self): + StepCollector.__init__(self, space=None) + self._state_transitions = iter([ + (SCANNING, MARKING), + (MARKING, SWEEPING), + (SWEEPING, FINALIZING), + (FINALIZING, SCANNING)]) + + def _collect_step(self): + self.my_steps += 1 + try: + oldstate, newstate = next(self._state_transitions) + except StopIteration: + assert False, 'should not happen, did you call _collect_step too much?' + return rgc._encode_states(oldstate, newstate) + + def _run_finalizers(self): + self.my_finalized += 1 + + sc = MyStepCollector() + transitions = [] + while True: + result = sc.do() + transitions.append((result.oldstate, result.newstate, sc.my_finalized)) + if result.major_is_done: + break + + assert transitions == [ + (SCANNING, MARKING, False), + (MARKING, SWEEPING, False), + (SWEEPING, FINALIZING, False), + (FINALIZING, USERDEL, False), + (USERDEL, SCANNING, True) + ] + # there is one more transition than actual step, because + # FINALIZING->USERDEL is "virtual" + assert sc.my_steps == len(transitions) - 1 diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -69,26 +69,29 @@ def test_on_gc_collect_step(self): import gc + SCANNING = 0 + MARKING = 1 + SWEEPING = 2 + FINALIZING = 3 lst = [] def on_gc_collect_step(stats): lst.append((stats.count, stats.duration, stats.oldstate, - stats.newstate)) + stats.newstate, + stats.major_is_done)) gc.hooks.on_gc_collect_step = on_gc_collect_step - self.fire_gc_collect_step(10, 20, 30) - self.fire_gc_collect_step(40, 50, 60) + self.fire_gc_collect_step(10, SCANNING, MARKING) + self.fire_gc_collect_step(40, FINALIZING, SCANNING) assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), + (1, 10, SCANNING, MARKING, False), + (1, 40, FINALIZING, SCANNING, True), ] # gc.hooks.on_gc_collect_step = None - self.fire_gc_collect_step(70, 80, 90) # won't fire - assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), - ] + oldlst = lst[:] + self.fire_gc_collect_step(70, SCANNING, MARKING) # won't fire + assert lst == oldlst def test_on_gc_collect(self): import gc @@ -123,7 +126,8 @@ assert S.STATE_MARKING == 1 assert S.STATE_SWEEPING == 2 assert S.STATE_FINALIZING == 3 - assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', + 'FINALIZING', 'USERDEL') def test_cumulative(self): import gc @@ -176,3 +180,22 @@ assert gc.hooks.on_gc_minor is None assert gc.hooks.on_gc_collect_step is None assert gc.hooks.on_gc_collect is None + + def test_no_recursive(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + self.fire_gc_minor(1, 2, 3) # won't fire NOW + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + # the duration for the 2nd call is 41, because it also counts the 1 + # which was fired recursively + assert lst == [ + (1, 10, 20, 30), + (2, 41, 50, 60), + ] diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -6,11 +6,6 @@ from rpython.rtyper.lltypesystem.module.test.math_cases import (MathTests, get_tester) -consistent_host = True -if '__pypy__' not in sys.builtin_module_names: - if sys.version_info < (2, 6): - consistent_host = False - class TestDirect(MathTests): pass @@ -30,8 +25,6 @@ def make_test_case((fnname, args, expected), dict): # def test_func(self): - if not consistent_host: - py.test.skip("inconsistent behavior before 2.6") try: fn = getattr(math, fnname) except AttributeError: diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -18,7 +18,6 @@ filename = filename[:-1] space = cls.space cls.w_math_cases = space.wrap(filename) - cls.w_consistent_host = space.wrap(test_direct.consistent_host) @classmethod def make_callable_wrapper(cls, func): @@ -53,8 +52,6 @@ yield fnname, args, expected def test_all_cases(self): - if not self.consistent_host: - skip("please test this on top of PyPy or CPython >= 2.6") import math for fnname, args, expected in self.cases(): fn = getattr(math, fnname) diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -149,6 +149,20 @@ def get_size_incl_hash(self, obj): return self.get_size(obj) + # these can be overriden by subclasses, called by the GCTransformer + def enable(self): + pass + + def disable(self): + pass + + def isenabled(self): + return True + + def collect_step(self): + self.collect() + return True + def malloc(self, typeid, length=0, zero=False): """NOT_RPYTHON For testing. The interface used by the gctransformer is diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -379,6 +379,11 @@ self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING + + # if the GC is disabled, it runs only minor collections; major + # collections need to be manually triggered by explicitly calling + # collect() + self.enabled = True # # Two lists of all objects with finalizers. Actually they are lists # of pairs (finalization_queue_nr, object). "probably young objects" @@ -514,6 +519,15 @@ bigobj = self.nonlarge_max + 1 self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + def enable(self): + self.enabled = True + + def disable(self): + self.enabled = False + + def isenabled(self): + return self.enabled + def _nursery_memory_size(self): extra = self.nonlarge_max + 1 return self.nursery_size + extra @@ -750,22 +764,53 @@ """Do a minor (gen=0), start a major (gen=1), or do a full major (gen>=2) collection.""" if gen < 0: - self._minor_collection() # dangerous! no major GC cycle progress - elif gen <= 1: - self.minor_collection_with_major_progress() - if gen == 1 and self.gc_state == STATE_SCANNING: + # Dangerous! this makes no progress on the major GC cycle. + # If called too often, the memory usage will keep increasing, + # because we'll never completely fill the nursery (and so + # never run anything about the major collection). + self._minor_collection() + elif gen == 0: + # This runs a minor collection. This is basically what occurs + # when the nursery is full. If a major collection is in + # progress, it also runs one more step of it. It might also + # decide to start a major collection just now, depending on + # current memory pressure. + self.minor_collection_with_major_progress(force_enabled=True) + elif gen == 1: + # This is like gen == 0, but if no major collection is running, + # then it forces one to start now. + self.minor_collection_with_major_progress(force_enabled=True) + if self.gc_state == STATE_SCANNING: self.major_collection_step() else: + # This does a complete minor and major collection. self.minor_and_major_collection() self.rrc_invoke_callback() + def collect_step(self): + """ + Do a single major collection step. Return True when the major collection + is completed. - def minor_collection_with_major_progress(self, extrasize=0): - """Do a minor collection. Then, if there is already a major GC - in progress, run at least one major collection step. If there is - no major GC but the threshold is reached, start a major GC. + This is meant to be used together with gc.disable(), to have a + fine-grained control on when the GC runs. + """ + old_state = self.gc_state + self._minor_collection() + self.major_collection_step() + self.rrc_invoke_callback() + return rgc._encode_states(old_state, self.gc_state) + + def minor_collection_with_major_progress(self, extrasize=0, + force_enabled=False): + """Do a minor collection. Then, if the GC is enabled and there + is already a major GC in progress, run at least one major collection + step. If there is no major GC but the threshold is reached, start a + major GC. """ self._minor_collection() + if not self.enabled and not force_enabled: + return # If the gc_state is STATE_SCANNING, we're not in the middle # of an incremental major collection. In that case, wait @@ -2428,25 +2473,6 @@ # We also need to reset the GCFLAG_VISITED on prebuilt GC objects. self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None) # - # Print statistics - debug_start("gc-collect-done") - debug_print("arenas: ", - self.stat_ac_arenas_count, " => ", - self.ac.arenas_count) - debug_print("bytes used in arenas: ", - self.ac.total_memory_used) - debug_print("bytes raw-malloced: ", - self.stat_rawmalloced_total_size, " => ", - self.rawmalloced_total_size) - debug_stop("gc-collect-done") - self.hooks.fire_gc_collect( - num_major_collects=self.num_major_collects, - arenas_count_before=self.stat_ac_arenas_count, - arenas_count_after=self.ac.arenas_count, - arenas_bytes=self.ac.total_memory_used, - rawmalloc_bytes_before=self.stat_rawmalloced_total_size, - rawmalloc_bytes_after=self.rawmalloced_total_size) - # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than # we currently have -- but no more than 'max_delta' more than @@ -2460,6 +2486,27 @@ total_memory_used + self.max_delta), reserving_size) # + # Print statistics + debug_start("gc-collect-done") + debug_print("arenas: ", + self.stat_ac_arenas_count, " => ", + self.ac.arenas_count) + debug_print("bytes used in arenas: ", + self.ac.total_memory_used) + debug_print("bytes raw-malloced: ", + self.stat_rawmalloced_total_size, " => ", + self.rawmalloced_total_size) + debug_print("next major collection threshold: ", + self.next_major_collection_threshold) + debug_stop("gc-collect-done") + self.hooks.fire_gc_collect( + num_major_collects=self.num_major_collects, + arenas_count_before=self.stat_ac_arenas_count, + arenas_count_after=self.ac.arenas_count, + arenas_bytes=self.ac.total_memory_used, + rawmalloc_bytes_before=self.stat_rawmalloced_total_size, + rawmalloc_bytes_after=self.rawmalloced_total_size) + # # Max heap size: gives an upper bound on the threshold. If we # already have at least this much allocated, raise MemoryError. if bounded and self.threshold_reached(reserving_size): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -13,6 +13,7 @@ from rpython.memory.gc import minimark, incminimark from rpython.memory.gctypelayout import zero_gc_pointers_inside, zero_gc_pointers from rpython.rlib.debug import debug_print +from rpython.rlib.test.test_debug import debuglog import pdb WORD = LONG_BIT // 8 @@ -770,4 +771,76 @@ assert elem.prev == lltype.nullptr(S) assert elem.next == lltype.nullptr(S) - + def test_collect_0(self, debuglog): + self.gc.collect(1) # start a major + debuglog.reset() + self.gc.collect(0) # do ONLY a minor + assert debuglog.summary() == {'gc-minor': 1} + + def test_enable_disable(self, debuglog): + def large_malloc(): + # malloc an object which is large enough to trigger a major collection + threshold = self.gc.next_major_collection_threshold + self.malloc(VAR, int(threshold/8)) + summary = debuglog.summary() + debuglog.reset() + return summary + # + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + # + self.gc.disable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-minor'] + # + self.gc.enable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + + def test_call_collect_when_disabled(self, debuglog): + # malloc an object and put it the old generation + s = self.malloc(S) + s.x = 42 + self.stackroots.append(s) + self.gc.collect() + s = self.stackroots.pop() + # + self.gc.disable() + self.gc.collect(1) # start a major collect + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect(1) # run one more step + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect() # finish the major collection + summary = debuglog.summary() + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + # s is freed + py.test.raises(RuntimeError, 's.x') + + def test_collect_step(self, debuglog): + from rpython.rlib import rgc + n = 0 + states = [] + while True: + debuglog.reset() + val = self.gc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + summary = debuglog.summary() + assert summary == {'gc-minor': 1, 'gc-collect-step': 1} + if rgc.is_done(val): + break + n += 1 + if n == 100: + assert False, 'this looks like an endless loop' + # + assert states == [ + (incminimark.STATE_SCANNING, incminimark.STATE_MARKING), + (incminimark.STATE_MARKING, incminimark.STATE_SWEEPING), + (incminimark.STATE_SWEEPING, incminimark.STATE_FINALIZING), + (incminimark.STATE_FINALIZING, incminimark.STATE_SCANNING) + ] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -309,6 +309,12 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) + self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc], + annmodel.SomeInteger()) + self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None) + self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None) + self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc], + annmodel.s_Bool) self.can_move_ptr = getfn(GCClass.can_move.im_func, [s_gc, SomeAddress()], annmodel.SomeBool()) @@ -884,6 +890,28 @@ resultvar=op.result) self.pop_roots(hop, livevars) + def gct_gc__collect_step(self, hop): + op = hop.spaceop + livevars = self.push_roots(hop) + hop.genop("direct_call", [self.collect_step_ptr, self.c_const_gc], + resultvar=op.result) + self.pop_roots(hop, livevars) + + def gct_gc__enable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.enable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__disable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.disable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__isenabled(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.isenabled_ptr, self.c_const_gc], + resultvar=op.result) + def gct_gc_can_move(self, hop): op = hop.spaceop v_addr = hop.genop('cast_ptr_to_adr', diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,5 +1,6 @@ import sys import time +from collections import Counter from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry @@ -38,6 +39,23 @@ assert False, ("nesting error: no start corresponding to stop %r" % (category,)) + def reset(self): + # only for tests: empty the log + self[:] = [] + + def summary(self, flatten=False): + res = Counter() + def visit(lst): + for section, sublist in lst: + if section == 'debug_print': + continue + res[section] += 1 + if flatten: + visit(sublist) + # + visit(self) + return res + def __repr__(self): import pprint return pprint.pformat(list(self)) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -13,6 +13,50 @@ # General GC features collect = gc.collect +enable = gc.enable +disable = gc.disable +isenabled = gc.isenabled + +def collect_step(): + """ + If the GC is incremental, run a single gc-collect-step. + + Return an integer which encodes the starting and ending GC state. Use + rgc.{old_state,new_state,is_done} to decode it. + + If the GC is not incremental, do a full collection and return a value on + which rgc.is_done() return True. + """ + gc.collect() + return _encode_states(1, 0) + +def _encode_states(oldstate, newstate): + return oldstate << 8 | newstate + +def old_state(states): + return (states & 0xFF00) >> 8 + +def new_state(states): + return states & 0xFF + +def is_done(states): + """ + Return True if the return value of collect_step signals the end of a major + collection + """ + old = old_state(states) + new = new_state(states) + return is_done__states(old, new) + +def is_done__states(oldstate, newstate): + "Like is_done, but takes oldstate and newstate explicitly" + # a collection is considered done when it ends up in the starting state + # (which is usually represented as 0). This logic works for incminimark, + # which is currently the only gc actually used and for which collect_step + # is implemented. In case we add more GC in the future, we might want to + # delegate this logic to the GC itself, but for now it is MUCH simpler to + # just write it in plain RPython. + return oldstate != 0 and newstate == 0 def set_max_heap_size(nbytes): """Limit the heap size to n bytes. @@ -131,6 +175,44 @@ args_v = hop.inputargs(lltype.Signed) return hop.genop('gc__collect', args_v, resulttype=hop.r_result) + +class EnableDisableEntry(ExtRegistryEntry): + _about_ = (gc.enable, gc.disable) + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + hop.exception_cannot_occur() + opname = self.instance.__name__ + return hop.genop('gc__%s' % opname, hop.args_v, resulttype=hop.r_result) + + +class IsEnabledEntry(ExtRegistryEntry): + _about_ = gc.isenabled + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_Bool + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result) + + +class CollectStepEntry(ExtRegistryEntry): + _about_ = collect_step + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.SomeInteger() + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__collect_step', hop.args_v, resulttype=hop.r_result) + + class SetMaxHeapSizeEntry(ExtRegistryEntry): _about_ = set_max_heap_size diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py --- a/rpython/rlib/test/test_debug.py +++ b/rpython/rlib/test/test_debug.py @@ -1,5 +1,5 @@ - import py +import pytest from rpython.rlib.debug import (check_annotation, make_sure_not_resized, debug_print, debug_start, debug_stop, have_debug_prints, debug_offset, debug_flush, @@ -10,6 +10,12 @@ from rpython.rlib import debug from rpython.rtyper.test.test_llinterp import interpret, gengraph + at pytest.fixture +def debuglog(monkeypatch): + dlog = debug.DebugLog() + monkeypatch.setattr(debug, '_log', dlog) + return dlog + def test_check_annotation(): class Error(Exception): pass @@ -94,7 +100,7 @@ py.test.raises(NotAListOfChars, "interpret(g, [3])") -def test_debug_print_start_stop(): +def test_debug_print_start_stop(debuglog): def f(x): debug_start("mycat") debug_print("foo", 2, "bar", x) @@ -103,22 +109,27 @@ debug_offset() # should not explode at least return have_debug_prints() - try: - debug._log = dlog = debug.DebugLog() - res = f(3) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = f(3) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + debuglog.reset() - try: - debug._log = dlog = debug.DebugLog() - res = interpret(f, [3]) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = interpret(f, [3]) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] +def test_debuglog_summary(debuglog): + debug_start('foo') + debug_start('bar') # this is nested, so not counted in the summary by default + debug_stop('bar') + debug_stop('foo') + debug_start('foo') + debug_stop('foo') + debug_start('bar') + debug_stop('bar') + # + assert debuglog.summary() == {'foo': 2, 'bar': 1} + assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2} def test_debug_start_stop_timestamp(): import time diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -39,6 +39,45 @@ assert res is None +def test_enable_disable(): + def f(): + gc.enable() + a = gc.isenabled() + gc.disable() + b = gc.isenabled() + return a and not b + + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__enable', 'gc__isenabled', + 'gc__disable', 'gc__isenabled'] + res = interpret(f, []) + assert res + +def test_collect_step(): + def f(): + return rgc.collect_step() + + assert f() + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__collect_step'] + res = interpret(f, []) + assert res + +def test__encode_states(): + val = rgc._encode_states(42, 43) + assert rgc.old_state(val) == 42 + assert rgc.new_state(val) == 43 + assert not rgc.is_done(val) + # + val = rgc.collect_step() + assert rgc.is_done(val) + def test_can_move(): T0 = lltype.GcStruct('T') T1 = lltype.GcArray(lltype.Float) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -819,6 +819,18 @@ def op_gc__collect(self, *gen): self.heap.collect(*gen) + def op_gc__collect_step(self): + return self.heap.collect_step() + + def op_gc__enable(self): + self.heap.enable() + + def op_gc__disable(self): + self.heap.disable() + + def op_gc__isenabled(self): + return self.heap.isenabled() + def op_gc_heap_stats(self): raise NotImplementedError diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py --- a/rpython/rtyper/lltypesystem/llheap.py +++ b/rpython/rtyper/lltypesystem/llheap.py @@ -5,7 +5,7 @@ setfield = setattr from operator import setitem as setarrayitem -from rpython.rlib.rgc import can_move, collect, add_memory_pressure +from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure, collect_step def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue, offsets=None): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -456,6 +456,10 @@ # __________ GC operations __________ 'gc__collect': LLOp(canmallocgc=True), + 'gc__collect_step': LLOp(canmallocgc=True), + 'gc__enable': LLOp(), + 'gc__disable': LLOp(), + 'gc__isenabled': LLOp(), 'gc_free': LLOp(), 'gc_fetch_exception': LLOp(), 'gc_restore_exception': LLOp(), diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,6 +1812,92 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_enable_disable(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(should_disable): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + if should_disable: + gc.disable() + assert not gc.isenabled() + # try to trigger a major collection + N = 100 # this should be enough, increase if not + lst = [] + for i in range(N): + lst.append(chr(i%256) * (1024*1024)) + #print i, counter.val + # + gc.enable() + assert gc.isenabled() + return counter.val + return f + + def test_enable_disable(self): + # first, run with normal gc. If the assert fails it means that in the + # loop we don't allocate enough mem to trigger a major collection. Try + # to increase N + deleted = self.run("enable_disable", 0) + assert deleted == 1, 'This should not fail, try to increment N' + # + # now, run with gc.disable: this should NOT free x1 + deleted = self.run("enable_disable", 1) + assert deleted == 0 + + def define_collect_step(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + gc.disable() + n = 0 + states = [] + while True: + n += 1 + val = rgc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + if rgc.is_done(val): + break + if n == 100: + print 'Endless loop!' + assert False, 'this looks like an endless loop' + + if n < 4: # we expect at least 4 steps + print 'Too few steps! n =', n + assert False + + # check that the state transitions are reasonable + first_state, _ = states[0] + for i, (old_state, new_state) in enumerate(states): + is_last = (i == len(states) - 1) + is_valid = False + if is_last: + assert old_state != new_state == first_state + else: + assert new_state == old_state or new_state == old_state+1 + + return counter.val + return f + + def test_collect_step(self): + deleted = self.run("collect_step") + assert deleted == 1 + def define_total_gc_time(cls): def f(): l = [] diff --git a/rpython/translator/goal/gcbench.py b/rpython/translator/goal/gcbench.py --- a/rpython/translator/goal/gcbench.py +++ b/rpython/translator/goal/gcbench.py @@ -44,8 +44,9 @@ # - Results are sensitive to locking cost, but we dont # check for proper locking import time +import gc -USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N]""" +USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N] [--gc=off|--gc=manual]""" ENABLE_THREADS = True @@ -173,6 +174,7 @@ depths = DEFAULT_DEPTHS threads = 0 repeatcount = 1 + gc_policy = 'on' for arg in argv[1:]: if arg.startswith('--threads='): arg = arg[len('--threads='):] @@ -189,13 +191,22 @@ depths = [int(s) for s in arg] except ValueError: return argerror() + elif arg.startswith('--gc=off'): + gc_policy = 'off' + elif arg.startswith('--gc=manual'): + gc_policy = 'manual' else: try: repeatcount = int(arg) except ValueError: return argerror() + # + if gc_policy == 'off' or gc_policy == 'manual': + gc.disable() for i in range(repeatcount): main(depths, threads) + if gc_policy == 'manual': + gc.collect(1) return 0 From pypy.commits at gmail.com Mon Dec 24 02:46:03 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 23 Dec 2018 23:46:03 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix exception type Message-ID: <5c208ebb.1c69fb81.2d099.36f5@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95521:528771eac6b5 Date: 2018-12-24 09:45 +0200 http://bitbucket.org/pypy/pypy/changeset/528771eac6b5/ Log: fix exception type diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -379,7 +379,7 @@ def _pow(space, iv, iw, iz): if iw < 0: if iz != 0: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_ValueError, "pow() 2nd argument cannot be negative when 3rd " "argument specified") raise ValueError From pypy.commits at gmail.com Mon Dec 24 10:28:14 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:14 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: always re-encode unexpected kwd argument Message-ID: <5c20fb0e.1c69fb81.50c42.ec6c@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95522:be8caf00c728 Date: 2018-12-24 12:24 +0200 http://bitbucket.org/pypy/pypy/changeset/be8caf00c728/ Log: always re-encode unexpected kwd argument diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -535,24 +535,23 @@ if num_remainingkwds == 1: for i in range(len(keywords)): if i not in kwds_mapping: - name = keywords[i] - if name is None: - # We'll assume it's unicode. Encode it. - # Careful, I *think* it should not be possible to - # get an IndexError here but you never know. - try: - if keyword_names_w is None: - raise IndexError - # note: negative-based indexing from the end - w_name = keyword_names_w[i - len(keywords)] - except IndexError: - name = '?' - else: - w_enc = space.newtext(space.sys.defaultencoding) - w_err = space.newtext("replace") - w_name = space.call_method(w_name, "encode", w_enc, - w_err) - name = space.text_w(w_name) + name = '?' + # We'll assume it's unicode. Encode it. + # Careful, I *think* it should not be possible to + # get an IndexError here but you never know. + try: + if keyword_names_w is None: + raise IndexError + # note: negative-based indexing from the end + w_name = keyword_names_w[i - len(keywords)] + except IndexError: + name = '?' + else: + w_enc = space.newtext(space.sys.defaultencoding) + w_err = space.newtext("replace") + w_name = space.call_method(w_name, "encode", w_enc, + w_err) + name = space.text_w(w_name) break self.kwd_name = name From pypy.commits at gmail.com Mon Dec 24 10:28:17 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:17 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py.35 into branch Message-ID: <5c20fb11.1c69fb81.50a0a.df27@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95523:25050ea5c0b4 Date: 2018-12-24 12:24 +0200 http://bitbucket.org/pypy/pypy/changeset/25050ea5c0b4/ Log: merge py.35 into branch diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -327,6 +327,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(self): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py --- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py +++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py @@ -244,6 +244,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_memmove(): ffi = _cffi1_backend.FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -1654,6 +1654,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_all_primitives(self): assert set(PRIMITIVE_TO_INDEX) == set([ diff --git a/extra_tests/test_pyrepl/conftest.py b/extra_tests/test_pyrepl/conftest.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/conftest.py @@ -0,0 +1,8 @@ +import sys + +def pytest_ignore_collect(path): + if '__pypy__' not in sys.builtin_module_names: + try: + import pyrepl + except ImportError: + return True diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -521,7 +521,11 @@ -self._microseconds) def __pos__(self): - return self + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days, + self._seconds, + self._microseconds) def __abs__(self): if self._days < 0: @@ -813,8 +817,7 @@ month = self._month if day is None: day = self._day - # PyPy fix: returns type(self)() instead of date() - return type(self)(year, month, day) + return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. @@ -1289,8 +1292,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - # PyPy fix: returns type(self)() instead of time() - return type(self)(hour, minute, second, microsecond, tzinfo) + return time.__new__(type(self), + hour, minute, second, microsecond, tzinfo) # Pickle support. @@ -1341,13 +1344,13 @@ hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = dateinterop.__new__(cls) - self._year = year - self._month = month - self._day = day - self._hour = hour - self._minute = minute - self._second = second - self._microsecond = microsecond + self._year = int(year) + self._month = int(month) + self._day = int(day) + self._hour = int(hour) + self._minute = int(minute) + self._second = int(second) + self._microsecond = int(microsecond) self._tzinfo = tzinfo self._hashcode = -1 return self @@ -1503,8 +1506,8 @@ if tzinfo is True: tzinfo = self.tzinfo # PyPy fix: returns type(self)() instead of datetime() - return type(self)(year, month, day, hour, minute, second, microsecond, - tzinfo) + return datetime.__new__(type(self), year, month, day, hour, minute, + second, microsecond, tzinfo) def astimezone(self, tz=None): if tz is None: @@ -1768,7 +1771,10 @@ if myoff == otoff: return base if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") + # The CPython _datetimemodule.c error message and the + # datetime.py one are different + raise TypeError("can't subtract offset-naive and " + "offset-aware datetimes") return base + otoff - myoff def __hash__(self): diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -341,7 +341,7 @@ # """ # note that 'buffer' is a type, set on this instance by __init__ - def from_buffer(self, python_buffer): + def from_buffer(self, python_buffer, require_writable=False): """Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is not meant to be used on the built-in types @@ -349,7 +349,8 @@ but only on objects containing large quantities of raw data in some other format, like 'array.array' or numpy arrays. """ - return self._backend.from_buffer(self.BCharA, python_buffer) + return self._backend.from_buffer(self.BCharA, python_buffer, + require_writable) def memmove(self, dest, src, n): """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -401,8 +401,10 @@ * some functions and attributes of the ``gc`` module behave in a slightly different way: for example, ``gc.enable`` and - ``gc.disable`` are supported, but instead of enabling and disabling - the GC, they just enable and disable the execution of finalizers. + ``gc.disable`` are supported, but "enabling and disabling the GC" has + a different meaning in PyPy than in CPython. These functions + actually enable and disable the major collections and the + execution of finalizers. * PyPy prints a random line from past #pypy IRC topics at startup in interactive mode. In a released version, this behaviour is suppressed, but diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -22,8 +22,44 @@ larger. (A third category, the very large objects, are initially allocated outside the nursery and never move.) -Since Incminimark is an incremental GC, the major collection is incremental, -meaning there should not be any pauses longer than 1ms. +Since Incminimark is an incremental GC, the major collection is incremental: +the goal is not to have any pause longer than 1ms, but in practice it depends +on the size and characteristics of the heap: occasionally, there can be pauses +between 10-100ms. + + +Semi-manual GC management +-------------------------- + +If there are parts of the program where it is important to have a low latency, +you might want to control precisely when the GC runs, to avoid unexpected +pauses. Note that this has effect only on major collections, while minor +collections continue to work as usual. + +As explained above, a full major collection consists of ``N`` steps, where +``N`` depends on the size of the heap; generally speaking, it is not possible +to predict how many steps will be needed to complete a collection. + +``gc.enable()`` and ``gc.disable()`` control whether the GC runs collection +steps automatically. When the GC is disabled the memory usage will grow +indefinitely, unless you manually call ``gc.collect()`` and +``gc.collect_step()``. + +``gc.collect()`` runs a full major collection. + +``gc.collect_step()`` runs a single collection step. It returns an object of +type GcCollectStepStats_, the same which is passed to the corresponding `GC +Hooks`_. The following code is roughly equivalent to a ``gc.collect()``:: + + while True: + if gc.collect_step().major_is_done: + break + +For a real-world example of usage of this API, you can look at the 3rd-party +module `pypytools.gc.custom`_, which also provides a ``with customgc.nogc()`` +context manager to mark sections where the GC is forbidden. + +.. _`pypytools.gc.custom`: https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default Fragmentation @@ -184,6 +220,8 @@ the number of pinned objects. +.. _GcCollectStepStats: + The attributes for ``GcCollectStepStats`` are: ``count``, ``duration``, ``duration_min``, ``duration_max`` @@ -192,10 +230,14 @@ ``oldstate``, ``newstate`` Integers which indicate the state of the GC before and after the step. +``major_is_done`` + Boolean which indicate whether this was the last step of the major + collection + The value of ``oldstate`` and ``newstate`` is one of these constants, defined inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, -``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string -representation of it by indexing the ``GC_STATS`` tuple. +``STATE_SWEEPING``, ``STATE_FINALIZING``, ``STATE_USERDEL``. It is possible +to get a string representation of it by indexing the ``GC_STATES`` tuple. The attributes for ``GcCollectStats`` are: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -67,4 +67,11 @@ .. branch: cleanup-test_lib_pypy -Update most test_lib_pypy/ tests and move them to extra_tests/. \ No newline at end of file +Update most test_lib_pypy/ tests and move them to extra_tests/. + +.. branch: gc-disable + +Make it possible to manually manage the GC by using a combination of +gc.disable() and gc.collect_step(). Make sure to write a proper release +announcement in which we explain that existing programs could leak memory if +they run for too much time between a gc.disable()/gc.enable() \ No newline at end of file diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -327,7 +327,8 @@ return w_ctype.cast(w_ob) - def descr_from_buffer(self, w_python_buffer): + @unwrap_spec(require_writable=int) + def descr_from_buffer(self, w_python_buffer, require_writable=0): """\ Return a that points to the data of the given Python object, which must support the buffer interface. Note that this is @@ -337,7 +338,8 @@ 'array.array' or numpy arrays.""" # w_ctchara = newtype._new_chara_type(self.space) - return func._from_buffer(self.space, w_ctchara, w_python_buffer) + return func._from_buffer(self.space, w_ctchara, w_python_buffer, + require_writable) @unwrap_spec(w_arg=W_CData) diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -110,8 +110,8 @@ def _fetch_as_write_buffer(space, w_x): return space.writebuf_w(w_x) - at unwrap_spec(w_ctype=ctypeobj.W_CType) -def from_buffer(space, w_ctype, w_x): + at unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int) +def from_buffer(space, w_ctype, w_x, require_writable=0): from pypy.module._cffi_backend import ctypearray, ctypeprim # if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or @@ -119,13 +119,16 @@ raise oefmt(space.w_TypeError, "needs 'char[]', got '%s'", w_ctype.name) # - return _from_buffer(space, w_ctype, w_x) + return _from_buffer(space, w_ctype, w_x, require_writable) -def _from_buffer(space, w_ctype, w_x): +def _from_buffer(space, w_ctype, w_x, require_writable): if space.isinstance_w(w_x, space.w_unicode): raise oefmt(space.w_TypeError, - "from_buffer() cannot return the address a unicode") - buf = _fetch_as_read_buffer(space, w_x) + "from_buffer() cannot return the address of a unicode object") + if require_writable: + buf = _fetch_as_write_buffer(space, w_x) + else: + buf = _fetch_as_read_buffer(space, w_x) if space.isinstance_w(w_x, space.w_bytes): _cdata = get_raw_address_of_string(space, w_x) else: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3730,6 +3730,18 @@ check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") +def test_from_buffer_require_writable(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p1 = from_buffer(BCharA, b"foo", False) + assert p1 == from_buffer(BCharA, b"foo", False) + py.test.raises((TypeError, BufferError), from_buffer, BCharA, b"foo", True) + ba = bytearray(b"foo") + p1 = from_buffer(BCharA, ba, True) + p1[0] = b"g" + assert ba == b"goo" + def test_memmove(): Short = new_primitive_type("short") ShortA = new_array_type(new_pointer_type(Short), None) diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -287,6 +287,16 @@ assert ffi.typeof(c) is ffi.typeof("char[]") ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + assert c == ffi.from_buffer(a, True) + assert c == ffi.from_buffer(a, require_writable=True) + # + p = ffi.from_buffer(b"abcd") + assert p[2] == b"c" + # + assert p == ffi.from_buffer(b"abcd", False) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True) + raises((TypeError, BufferError), ffi.from_buffer, b"abcd", + require_writable=True) def test_from_buffer_BytesIO(self): from _cffi_backend import FFI diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -15,7 +15,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) @@ -25,7 +25,7 @@ PLATFORM := $(shell uname -s) ifeq ($(PLATFORM),Darwin) - cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup + cppflags+=-dynamiclib -single_module -undefined dynamic_lookup endif diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -4,6 +4,7 @@ class Module(MixedModule): interpleveldefs = { 'collect': 'interp_gc.collect', + 'collect_step': 'interp_gc.collect_step', 'enable': 'interp_gc.enable', 'disable': 'interp_gc.disable', 'isenabled': 'interp_gc.isenabled', diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,5 +1,6 @@ from rpython.memory.gc.hook import GcHooks -from rpython.memory.gc import incminimark +from rpython.memory.gc import incminimark +from rpython.rlib import rgc from rpython.rlib.nonconst import NonConstant from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault @@ -117,12 +118,24 @@ self.descr_set_on_gc_collect(space, space.w_None) -class GcMinorHookAction(AsyncAction): +class NoRecursiveAction(AsyncAction): + depth = 0 + + def perform(self, ec, frame): + if self.depth == 0: + try: + self.depth += 1 + return self._do_perform(ec, frame) + finally: + self.depth -= 1 + + +class GcMinorHookAction(NoRecursiveAction): total_memory_used = 0 pinned_objects = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -145,7 +158,7 @@ self.pinned_objects = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcMinorStats( self.count, self.duration, @@ -157,12 +170,12 @@ self.space.call_function(self.w_callable, w_stats) -class GcCollectStepHookAction(AsyncAction): +class GcCollectStepHookAction(NoRecursiveAction): oldstate = 0 newstate = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -185,19 +198,20 @@ self.newstate = NonConstant(-42) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStepStats( self.count, self.duration, self.duration_min, self.duration_max, self.oldstate, - self.newstate) + self.newstate, + rgc.is_done__states(self.oldstate, self.newstate)) self.reset() self.space.call_function(self.w_callable, w_stats) -class GcCollectHookAction(AsyncAction): +class GcCollectHookAction(NoRecursiveAction): num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -206,7 +220,7 @@ rawmalloc_bytes_after = 0 def __init__(self, space): - AsyncAction.__init__(self, space) + NoRecursiveAction.__init__(self, space) self.w_callable = space.w_None self.reset() @@ -227,7 +241,7 @@ self.rawmalloc_bytes_after = NonConstant(r_uint(42)) self.fire() - def perform(self, ec, frame): + def _do_perform(self, ec, frame): w_stats = W_GcCollectStats(self.count, self.num_major_collects, self.arenas_count_before, @@ -252,15 +266,32 @@ class W_GcCollectStepStats(W_Root): + # NOTE: this is specific to incminimark: if we want to integrate the + # applevel gc module with another gc, we probably need a more general + # approach to this. + # + # incminimark has 4 GC states: scanning, marking, sweeping and + # finalizing. However, from the user point of view, we have an additional + # "virtual" state: USERDEL, which represent when we run applevel + # finalizers after having completed a GC major collection. This state is + # never explicitly visible when using hooks, but it is used for the return + # value of gc.collect_step (see interp_gc.py) + STATE_SCANNING = incminimark.STATE_SCANNING + STATE_MARKING = incminimark.STATE_MARKING + STATE_SWEEPING = incminimark.STATE_SWEEPING + STATE_FINALIZING = incminimark.STATE_FINALIZING + STATE_USERDEL = incminimark.STATE_FINALIZING + 1 # used by StepCollector + GC_STATES = tuple(incminimark.GC_STATES + ['USERDEL']) def __init__(self, count, duration, duration_min, duration_max, - oldstate, newstate): + oldstate, newstate, major_is_done): self.count = count self.duration = duration self.duration_min = duration_min self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate + self.major_is_done = major_is_done class W_GcCollectStats(W_Root): @@ -320,11 +351,16 @@ W_GcCollectStepStats.typedef = TypeDef( "GcCollectStepStats", - STATE_SCANNING = incminimark.STATE_SCANNING, - STATE_MARKING = incminimark.STATE_MARKING, - STATE_SWEEPING = incminimark.STATE_SWEEPING, - STATE_FINALIZING = incminimark.STATE_FINALIZING, - GC_STATES = tuple(incminimark.GC_STATES), + STATE_SCANNING = W_GcCollectStepStats.STATE_SCANNING, + STATE_MARKING = W_GcCollectStepStats.STATE_MARKING, + STATE_SWEEPING = W_GcCollectStepStats.STATE_SWEEPING, + STATE_FINALIZING = W_GcCollectStepStats.STATE_FINALIZING, + STATE_USERDEL = W_GcCollectStepStats.STATE_USERDEL, + GC_STATES = tuple(W_GcCollectStepStats.GC_STATES), + major_is_done = interp_attrproperty( + "major_is_done", + cls=W_GcCollectStepStats, + wrapfn="newbool"), **wrap_many(W_GcCollectStepStats, ( "count", "duration", diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import oefmt from rpython.rlib import rgc +from pypy.module.gc.hook import W_GcCollectStepStats @unwrap_spec(generation=int) @@ -16,7 +17,9 @@ cache.clear() rgc.collect() + _run_finalizers(space) +def _run_finalizers(space): # if we are running in gc.disable() mode but gc.collect() is called, # we should still call the finalizers now. We do this as an attempt # to get closer to CPython's behavior: in Py3.5 some tests @@ -39,18 +42,20 @@ return space.newint(0) def enable(space): - """Non-recursive version. Enable finalizers now. + """Non-recursive version. Enable major collections and finalizers. If they were already enabled, no-op. If they were disabled even several times, enable them anyway. """ + rgc.enable() if not space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = True enable_finalizers(space) def disable(space): - """Non-recursive version. Disable finalizers now. Several calls - to this function are ignored. + """Non-recursive version. Disable major collections and finalizers. + Multiple calls to this function are ignored. """ + rgc.disable() if space.user_del_action.enabled_at_app_level: space.user_del_action.enabled_at_app_level = False disable_finalizers(space) @@ -77,6 +82,59 @@ if uda.pending_with_disabled_del is None: uda.pending_with_disabled_del = [] + +class StepCollector(object): + """ + Invoke rgc.collect_step() until we are done, then run the app-level + finalizers as a separate step + """ + + def __init__(self, space): + self.space = space + self.finalizing = False + + def do(self): + if self.finalizing: + self._run_finalizers() + self.finalizing = False + oldstate = W_GcCollectStepStats.STATE_USERDEL + newstate = W_GcCollectStepStats.STATE_SCANNING + major_is_done = True # now we are finally done + else: + states = self._collect_step() + oldstate = rgc.old_state(states) + newstate = rgc.new_state(states) + major_is_done = False # USERDEL still to do + if rgc.is_done(states): + newstate = W_GcCollectStepStats.STATE_USERDEL + self.finalizing = True + # + duration = -1 + return W_GcCollectStepStats( + count = 1, + duration = duration, + duration_min = duration, + duration_max = duration, + oldstate = oldstate, + newstate = newstate, + major_is_done = major_is_done) + + def _collect_step(self): + return rgc.collect_step() + + def _run_finalizers(self): + _run_finalizers(self.space) + +def collect_step(space): + """ + If the GC is incremental, run a single gc-collect-step. Return True when + the major collection is completed. + If the GC is not incremental, do a full collection and return True. + """ + sc = space.fromcache(StepCollector) + w_stats = sc.do() + return w_stats + # ____________________________________________________________ @unwrap_spec(filename='fsencode') diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -1,7 +1,20 @@ import py - +import pytest +from rpython.rlib import rgc +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.module.gc.interp_gc import StepCollector, W_GcCollectStepStats class AppTestGC(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + def rgc_isenabled(space): + return space.newbool(rgc.isenabled()) + cls.w_rgc_isenabled = space.wrap(interp2app(rgc_isenabled)) + def test_collect(self): import gc gc.collect() # mostly a "does not crash" kind of test @@ -63,12 +76,16 @@ def test_enable(self): import gc assert gc.isenabled() + assert self.rgc_isenabled() gc.disable() assert not gc.isenabled() + assert not self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() gc.enable() assert gc.isenabled() + assert self.rgc_isenabled() def test_gc_collect_overrides_gc_disable(self): import gc @@ -83,6 +100,24 @@ assert deleted == [1] gc.enable() + def test_gc_collect_step(self): + import gc + + class X(object): + deleted = 0 + def __del__(self): + X.deleted += 1 + + gc.disable() + X(); X(); X(); + n = 0 + while True: + n += 1 + if gc.collect_step().major_is_done: + break + + assert n >= 2 # at least one step + 1 finalizing + assert X.deleted == 3 class AppTestGcDumpHeap(object): pytestmark = py.test.mark.xfail(run=False) @@ -156,3 +191,55 @@ gc.collect() # the classes C should all go away here for r in rlist: assert r() is None + + +def test_StepCollector(): + W = W_GcCollectStepStats + SCANNING = W.STATE_SCANNING + MARKING = W.STATE_MARKING + SWEEPING = W.STATE_SWEEPING + FINALIZING = W.STATE_FINALIZING + USERDEL = W.STATE_USERDEL + + class MyStepCollector(StepCollector): + my_steps = 0 + my_done = False + my_finalized = 0 + + def __init__(self): + StepCollector.__init__(self, space=None) + self._state_transitions = iter([ + (SCANNING, MARKING), + (MARKING, SWEEPING), + (SWEEPING, FINALIZING), + (FINALIZING, SCANNING)]) + + def _collect_step(self): + self.my_steps += 1 + try: + oldstate, newstate = next(self._state_transitions) + except StopIteration: + assert False, 'should not happen, did you call _collect_step too much?' + return rgc._encode_states(oldstate, newstate) + + def _run_finalizers(self): + self.my_finalized += 1 + + sc = MyStepCollector() + transitions = [] + while True: + result = sc.do() + transitions.append((result.oldstate, result.newstate, sc.my_finalized)) + if result.major_is_done: + break + + assert transitions == [ + (SCANNING, MARKING, False), + (MARKING, SWEEPING, False), + (SWEEPING, FINALIZING, False), + (FINALIZING, USERDEL, False), + (USERDEL, SCANNING, True) + ] + # there is one more transition than actual step, because + # FINALIZING->USERDEL is "virtual" + assert sc.my_steps == len(transitions) - 1 diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -69,26 +69,29 @@ def test_on_gc_collect_step(self): import gc + SCANNING = 0 + MARKING = 1 + SWEEPING = 2 + FINALIZING = 3 lst = [] def on_gc_collect_step(stats): lst.append((stats.count, stats.duration, stats.oldstate, - stats.newstate)) + stats.newstate, + stats.major_is_done)) gc.hooks.on_gc_collect_step = on_gc_collect_step - self.fire_gc_collect_step(10, 20, 30) - self.fire_gc_collect_step(40, 50, 60) + self.fire_gc_collect_step(10, SCANNING, MARKING) + self.fire_gc_collect_step(40, FINALIZING, SCANNING) assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), + (1, 10, SCANNING, MARKING, False), + (1, 40, FINALIZING, SCANNING, True), ] # gc.hooks.on_gc_collect_step = None - self.fire_gc_collect_step(70, 80, 90) # won't fire - assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), - ] + oldlst = lst[:] + self.fire_gc_collect_step(70, SCANNING, MARKING) # won't fire + assert lst == oldlst def test_on_gc_collect(self): import gc @@ -123,7 +126,8 @@ assert S.STATE_MARKING == 1 assert S.STATE_SWEEPING == 2 assert S.STATE_FINALIZING == 3 - assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', + 'FINALIZING', 'USERDEL') def test_cumulative(self): import gc @@ -176,3 +180,22 @@ assert gc.hooks.on_gc_minor is None assert gc.hooks.on_gc_collect_step is None assert gc.hooks.on_gc_collect is None + + def test_no_recursive(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + self.fire_gc_minor(1, 2, 3) # won't fire NOW + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + # the duration for the 2nd call is 41, because it also counts the 1 + # which was fired recursively + assert lst == [ + (1, 10, 20, 30), + (2, 41, 50, 60), + ] diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -6,11 +6,6 @@ from rpython.rtyper.lltypesystem.module.test.math_cases import (MathTests, get_tester) -consistent_host = True -if '__pypy__' not in sys.builtin_module_names: - if sys.version_info < (2, 6): - consistent_host = False - class TestDirect(MathTests): pass @@ -30,8 +25,6 @@ def make_test_case((fnname, args, expected), dict): # def test_func(self): - if not consistent_host: - py.test.skip("inconsistent behavior before 2.6") try: fn = getattr(math, fnname) except AttributeError: diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -18,7 +18,6 @@ filename = filename[:-1] space = cls.space cls.w_math_cases = space.wrap(filename) - cls.w_consistent_host = space.wrap(test_direct.consistent_host) @classmethod def make_callable_wrapper(cls, func): @@ -53,8 +52,6 @@ yield fnname, args, expected def test_all_cases(self): - if not self.consistent_host: - skip("please test this on top of PyPy or CPython >= 2.6") import math for fnname, args, expected in self.cases(): fn = getattr(math, fnname) diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py --- a/pypy/objspace/std/smalllongobject.py +++ b/pypy/objspace/std/smalllongobject.py @@ -379,7 +379,7 @@ def _pow(space, iv, iw, iz): if iw < 0: if iz != 0: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_ValueError, "pow() 2nd argument cannot be negative when 3rd " "argument specified") raise ValueError diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -149,6 +149,20 @@ def get_size_incl_hash(self, obj): return self.get_size(obj) + # these can be overriden by subclasses, called by the GCTransformer + def enable(self): + pass + + def disable(self): + pass + + def isenabled(self): + return True + + def collect_step(self): + self.collect() + return True + def malloc(self, typeid, length=0, zero=False): """NOT_RPYTHON For testing. The interface used by the gctransformer is diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -379,6 +379,11 @@ self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING + + # if the GC is disabled, it runs only minor collections; major + # collections need to be manually triggered by explicitly calling + # collect() + self.enabled = True # # Two lists of all objects with finalizers. Actually they are lists # of pairs (finalization_queue_nr, object). "probably young objects" @@ -514,6 +519,15 @@ bigobj = self.nonlarge_max + 1 self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + def enable(self): + self.enabled = True + + def disable(self): + self.enabled = False + + def isenabled(self): + return self.enabled + def _nursery_memory_size(self): extra = self.nonlarge_max + 1 return self.nursery_size + extra @@ -750,22 +764,53 @@ """Do a minor (gen=0), start a major (gen=1), or do a full major (gen>=2) collection.""" if gen < 0: - self._minor_collection() # dangerous! no major GC cycle progress - elif gen <= 1: - self.minor_collection_with_major_progress() - if gen == 1 and self.gc_state == STATE_SCANNING: + # Dangerous! this makes no progress on the major GC cycle. + # If called too often, the memory usage will keep increasing, + # because we'll never completely fill the nursery (and so + # never run anything about the major collection). + self._minor_collection() + elif gen == 0: + # This runs a minor collection. This is basically what occurs + # when the nursery is full. If a major collection is in + # progress, it also runs one more step of it. It might also + # decide to start a major collection just now, depending on + # current memory pressure. + self.minor_collection_with_major_progress(force_enabled=True) + elif gen == 1: + # This is like gen == 0, but if no major collection is running, + # then it forces one to start now. + self.minor_collection_with_major_progress(force_enabled=True) + if self.gc_state == STATE_SCANNING: self.major_collection_step() else: + # This does a complete minor and major collection. self.minor_and_major_collection() self.rrc_invoke_callback() + def collect_step(self): + """ + Do a single major collection step. Return True when the major collection + is completed. - def minor_collection_with_major_progress(self, extrasize=0): - """Do a minor collection. Then, if there is already a major GC - in progress, run at least one major collection step. If there is - no major GC but the threshold is reached, start a major GC. + This is meant to be used together with gc.disable(), to have a + fine-grained control on when the GC runs. + """ + old_state = self.gc_state + self._minor_collection() + self.major_collection_step() + self.rrc_invoke_callback() + return rgc._encode_states(old_state, self.gc_state) + + def minor_collection_with_major_progress(self, extrasize=0, + force_enabled=False): + """Do a minor collection. Then, if the GC is enabled and there + is already a major GC in progress, run at least one major collection + step. If there is no major GC but the threshold is reached, start a + major GC. """ self._minor_collection() + if not self.enabled and not force_enabled: + return # If the gc_state is STATE_SCANNING, we're not in the middle # of an incremental major collection. In that case, wait @@ -2428,25 +2473,6 @@ # We also need to reset the GCFLAG_VISITED on prebuilt GC objects. self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None) # - # Print statistics - debug_start("gc-collect-done") - debug_print("arenas: ", - self.stat_ac_arenas_count, " => ", - self.ac.arenas_count) - debug_print("bytes used in arenas: ", - self.ac.total_memory_used) - debug_print("bytes raw-malloced: ", - self.stat_rawmalloced_total_size, " => ", - self.rawmalloced_total_size) - debug_stop("gc-collect-done") - self.hooks.fire_gc_collect( - num_major_collects=self.num_major_collects, - arenas_count_before=self.stat_ac_arenas_count, - arenas_count_after=self.ac.arenas_count, - arenas_bytes=self.ac.total_memory_used, - rawmalloc_bytes_before=self.stat_rawmalloced_total_size, - rawmalloc_bytes_after=self.rawmalloced_total_size) - # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than # we currently have -- but no more than 'max_delta' more than @@ -2460,6 +2486,27 @@ total_memory_used + self.max_delta), reserving_size) # + # Print statistics + debug_start("gc-collect-done") + debug_print("arenas: ", + self.stat_ac_arenas_count, " => ", + self.ac.arenas_count) + debug_print("bytes used in arenas: ", + self.ac.total_memory_used) + debug_print("bytes raw-malloced: ", + self.stat_rawmalloced_total_size, " => ", + self.rawmalloced_total_size) + debug_print("next major collection threshold: ", + self.next_major_collection_threshold) + debug_stop("gc-collect-done") + self.hooks.fire_gc_collect( + num_major_collects=self.num_major_collects, + arenas_count_before=self.stat_ac_arenas_count, + arenas_count_after=self.ac.arenas_count, + arenas_bytes=self.ac.total_memory_used, + rawmalloc_bytes_before=self.stat_rawmalloced_total_size, + rawmalloc_bytes_after=self.rawmalloced_total_size) + # # Max heap size: gives an upper bound on the threshold. If we # already have at least this much allocated, raise MemoryError. if bounded and self.threshold_reached(reserving_size): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -13,6 +13,7 @@ from rpython.memory.gc import minimark, incminimark from rpython.memory.gctypelayout import zero_gc_pointers_inside, zero_gc_pointers from rpython.rlib.debug import debug_print +from rpython.rlib.test.test_debug import debuglog import pdb WORD = LONG_BIT // 8 @@ -770,4 +771,76 @@ assert elem.prev == lltype.nullptr(S) assert elem.next == lltype.nullptr(S) - + def test_collect_0(self, debuglog): + self.gc.collect(1) # start a major + debuglog.reset() + self.gc.collect(0) # do ONLY a minor + assert debuglog.summary() == {'gc-minor': 1} + + def test_enable_disable(self, debuglog): + def large_malloc(): + # malloc an object which is large enough to trigger a major collection + threshold = self.gc.next_major_collection_threshold + self.malloc(VAR, int(threshold/8)) + summary = debuglog.summary() + debuglog.reset() + return summary + # + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + # + self.gc.disable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-minor'] + # + self.gc.enable() + summary = large_malloc() + assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor'] + + def test_call_collect_when_disabled(self, debuglog): + # malloc an object and put it the old generation + s = self.malloc(S) + s.x = 42 + self.stackroots.append(s) + self.gc.collect() + s = self.stackroots.pop() + # + self.gc.disable() + self.gc.collect(1) # start a major collect + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect(1) # run one more step + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + assert s.x == 42 # s is not freed yet + # + debuglog.reset() + self.gc.collect() # finish the major collection + summary = debuglog.summary() + assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor'] + # s is freed + py.test.raises(RuntimeError, 's.x') + + def test_collect_step(self, debuglog): + from rpython.rlib import rgc + n = 0 + states = [] + while True: + debuglog.reset() + val = self.gc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + summary = debuglog.summary() + assert summary == {'gc-minor': 1, 'gc-collect-step': 1} + if rgc.is_done(val): + break + n += 1 + if n == 100: + assert False, 'this looks like an endless loop' + # + assert states == [ + (incminimark.STATE_SCANNING, incminimark.STATE_MARKING), + (incminimark.STATE_MARKING, incminimark.STATE_SWEEPING), + (incminimark.STATE_SWEEPING, incminimark.STATE_FINALIZING), + (incminimark.STATE_FINALIZING, incminimark.STATE_SCANNING) + ] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -309,6 +309,12 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) + self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc], + annmodel.SomeInteger()) + self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None) + self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None) + self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc], + annmodel.s_Bool) self.can_move_ptr = getfn(GCClass.can_move.im_func, [s_gc, SomeAddress()], annmodel.SomeBool()) @@ -884,6 +890,28 @@ resultvar=op.result) self.pop_roots(hop, livevars) + def gct_gc__collect_step(self, hop): + op = hop.spaceop + livevars = self.push_roots(hop) + hop.genop("direct_call", [self.collect_step_ptr, self.c_const_gc], + resultvar=op.result) + self.pop_roots(hop, livevars) + + def gct_gc__enable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.enable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__disable(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.disable_ptr, self.c_const_gc], + resultvar=op.result) + + def gct_gc__isenabled(self, hop): + op = hop.spaceop + hop.genop("direct_call", [self.isenabled_ptr, self.c_const_gc], + resultvar=op.result) + def gct_gc_can_move(self, hop): op = hop.spaceop v_addr = hop.genop('cast_ptr_to_adr', diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,5 +1,6 @@ import sys import time +from collections import Counter from rpython.rlib.objectmodel import enforceargs from rpython.rtyper.extregistry import ExtRegistryEntry @@ -38,6 +39,23 @@ assert False, ("nesting error: no start corresponding to stop %r" % (category,)) + def reset(self): + # only for tests: empty the log + self[:] = [] + + def summary(self, flatten=False): + res = Counter() + def visit(lst): + for section, sublist in lst: + if section == 'debug_print': + continue + res[section] += 1 + if flatten: + visit(sublist) + # + visit(self) + return res + def __repr__(self): import pprint return pprint.pformat(list(self)) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -13,6 +13,50 @@ # General GC features collect = gc.collect +enable = gc.enable +disable = gc.disable +isenabled = gc.isenabled + +def collect_step(): + """ + If the GC is incremental, run a single gc-collect-step. + + Return an integer which encodes the starting and ending GC state. Use + rgc.{old_state,new_state,is_done} to decode it. + + If the GC is not incremental, do a full collection and return a value on + which rgc.is_done() return True. + """ + gc.collect() + return _encode_states(1, 0) + +def _encode_states(oldstate, newstate): + return oldstate << 8 | newstate + +def old_state(states): + return (states & 0xFF00) >> 8 + +def new_state(states): + return states & 0xFF + +def is_done(states): + """ + Return True if the return value of collect_step signals the end of a major + collection + """ + old = old_state(states) + new = new_state(states) + return is_done__states(old, new) + +def is_done__states(oldstate, newstate): + "Like is_done, but takes oldstate and newstate explicitly" + # a collection is considered done when it ends up in the starting state + # (which is usually represented as 0). This logic works for incminimark, + # which is currently the only gc actually used and for which collect_step + # is implemented. In case we add more GC in the future, we might want to + # delegate this logic to the GC itself, but for now it is MUCH simpler to + # just write it in plain RPython. + return oldstate != 0 and newstate == 0 def set_max_heap_size(nbytes): """Limit the heap size to n bytes. @@ -131,6 +175,44 @@ args_v = hop.inputargs(lltype.Signed) return hop.genop('gc__collect', args_v, resulttype=hop.r_result) + +class EnableDisableEntry(ExtRegistryEntry): + _about_ = (gc.enable, gc.disable) + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_None + + def specialize_call(self, hop): + hop.exception_cannot_occur() + opname = self.instance.__name__ + return hop.genop('gc__%s' % opname, hop.args_v, resulttype=hop.r_result) + + +class IsEnabledEntry(ExtRegistryEntry): + _about_ = gc.isenabled + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.s_Bool + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result) + + +class CollectStepEntry(ExtRegistryEntry): + _about_ = collect_step + + def compute_result_annotation(self): + from rpython.annotator import model as annmodel + return annmodel.SomeInteger() + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop('gc__collect_step', hop.args_v, resulttype=hop.r_result) + + class SetMaxHeapSizeEntry(ExtRegistryEntry): _about_ = set_max_heap_size diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py --- a/rpython/rlib/test/test_debug.py +++ b/rpython/rlib/test/test_debug.py @@ -1,5 +1,5 @@ - import py +import pytest from rpython.rlib.debug import (check_annotation, make_sure_not_resized, debug_print, debug_start, debug_stop, have_debug_prints, debug_offset, debug_flush, @@ -10,6 +10,12 @@ from rpython.rlib import debug from rpython.rtyper.test.test_llinterp import interpret, gengraph + at pytest.fixture +def debuglog(monkeypatch): + dlog = debug.DebugLog() + monkeypatch.setattr(debug, '_log', dlog) + return dlog + def test_check_annotation(): class Error(Exception): pass @@ -94,7 +100,7 @@ py.test.raises(NotAListOfChars, "interpret(g, [3])") -def test_debug_print_start_stop(): +def test_debug_print_start_stop(debuglog): def f(x): debug_start("mycat") debug_print("foo", 2, "bar", x) @@ -103,22 +109,27 @@ debug_offset() # should not explode at least return have_debug_prints() - try: - debug._log = dlog = debug.DebugLog() - res = f(3) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = f(3) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + debuglog.reset() - try: - debug._log = dlog = debug.DebugLog() - res = interpret(f, [3]) - assert res is True - finally: - debug._log = None - assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] + res = interpret(f, [3]) + assert res is True + assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] +def test_debuglog_summary(debuglog): + debug_start('foo') + debug_start('bar') # this is nested, so not counted in the summary by default + debug_stop('bar') + debug_stop('foo') + debug_start('foo') + debug_stop('foo') + debug_start('bar') + debug_stop('bar') + # + assert debuglog.summary() == {'foo': 2, 'bar': 1} + assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2} def test_debug_start_stop_timestamp(): import time diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -39,6 +39,45 @@ assert res is None +def test_enable_disable(): + def f(): + gc.enable() + a = gc.isenabled() + gc.disable() + b = gc.isenabled() + return a and not b + + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__enable', 'gc__isenabled', + 'gc__disable', 'gc__isenabled'] + res = interpret(f, []) + assert res + +def test_collect_step(): + def f(): + return rgc.collect_step() + + assert f() + t, typer, graph = gengraph(f, []) + blockops = list(graph.iterblockops()) + opnames = [op.opname for block, op in blockops + if op.opname.startswith('gc__')] + assert opnames == ['gc__collect_step'] + res = interpret(f, []) + assert res + +def test__encode_states(): + val = rgc._encode_states(42, 43) + assert rgc.old_state(val) == 42 + assert rgc.new_state(val) == 43 + assert not rgc.is_done(val) + # + val = rgc.collect_step() + assert rgc.is_done(val) + def test_can_move(): T0 = lltype.GcStruct('T') T1 = lltype.GcArray(lltype.Float) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -819,6 +819,18 @@ def op_gc__collect(self, *gen): self.heap.collect(*gen) + def op_gc__collect_step(self): + return self.heap.collect_step() + + def op_gc__enable(self): + self.heap.enable() + + def op_gc__disable(self): + self.heap.disable() + + def op_gc__isenabled(self): + return self.heap.isenabled() + def op_gc_heap_stats(self): raise NotImplementedError diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py --- a/rpython/rtyper/lltypesystem/llheap.py +++ b/rpython/rtyper/lltypesystem/llheap.py @@ -5,7 +5,7 @@ setfield = setattr from operator import setitem as setarrayitem -from rpython.rlib.rgc import can_move, collect, add_memory_pressure +from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure, collect_step def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue, offsets=None): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -456,6 +456,10 @@ # __________ GC operations __________ 'gc__collect': LLOp(canmallocgc=True), + 'gc__collect_step': LLOp(canmallocgc=True), + 'gc__enable': LLOp(), + 'gc__disable': LLOp(), + 'gc__isenabled': LLOp(), 'gc_free': LLOp(), 'gc_fetch_exception': LLOp(), 'gc_restore_exception': LLOp(), diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,6 +1812,92 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_enable_disable(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(should_disable): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + if should_disable: + gc.disable() + assert not gc.isenabled() + # try to trigger a major collection + N = 100 # this should be enough, increase if not + lst = [] + for i in range(N): + lst.append(chr(i%256) * (1024*1024)) + #print i, counter.val + # + gc.enable() + assert gc.isenabled() + return counter.val + return f + + def test_enable_disable(self): + # first, run with normal gc. If the assert fails it means that in the + # loop we don't allocate enough mem to trigger a major collection. Try + # to increase N + deleted = self.run("enable_disable", 0) + assert deleted == 1, 'This should not fail, try to increment N' + # + # now, run with gc.disable: this should NOT free x1 + deleted = self.run("enable_disable", 1) + assert deleted == 0 + + def define_collect_step(self): + class Counter(object): + val = 0 + counter = Counter() + class X(object): + def __del__(self): + counter.val += 1 + def f(): + x1 = X() + rgc.collect() # make x1 old + assert not rgc.can_move(x1) + x1 = None + # + gc.disable() + n = 0 + states = [] + while True: + n += 1 + val = rgc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + if rgc.is_done(val): + break + if n == 100: + print 'Endless loop!' + assert False, 'this looks like an endless loop' + + if n < 4: # we expect at least 4 steps + print 'Too few steps! n =', n + assert False + + # check that the state transitions are reasonable + first_state, _ = states[0] + for i, (old_state, new_state) in enumerate(states): + is_last = (i == len(states) - 1) + is_valid = False + if is_last: + assert old_state != new_state == first_state + else: + assert new_state == old_state or new_state == old_state+1 + + return counter.val + return f + + def test_collect_step(self): + deleted = self.run("collect_step") + assert deleted == 1 + def define_total_gc_time(cls): def f(): l = [] diff --git a/rpython/translator/goal/gcbench.py b/rpython/translator/goal/gcbench.py --- a/rpython/translator/goal/gcbench.py +++ b/rpython/translator/goal/gcbench.py @@ -44,8 +44,9 @@ # - Results are sensitive to locking cost, but we dont # check for proper locking import time +import gc -USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N]""" +USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N] [--gc=off|--gc=manual]""" ENABLE_THREADS = True @@ -173,6 +174,7 @@ depths = DEFAULT_DEPTHS threads = 0 repeatcount = 1 + gc_policy = 'on' for arg in argv[1:]: if arg.startswith('--threads='): arg = arg[len('--threads='):] @@ -189,13 +191,22 @@ depths = [int(s) for s in arg] except ValueError: return argerror() + elif arg.startswith('--gc=off'): + gc_policy = 'off' + elif arg.startswith('--gc=manual'): + gc_policy = 'manual' else: try: repeatcount = int(arg) except ValueError: return argerror() + # + if gc_policy == 'off' or gc_policy == 'manual': + gc.disable() for i in range(repeatcount): main(depths, threads) + if gc_policy == 'manual': + gc.collect(1) return 0 From pypy.commits at gmail.com Mon Dec 24 10:28:19 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:19 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: uncomment passing test Message-ID: <5c20fb13.1c69fb81.592de.7600@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95524:8968b3201bcd Date: 2018-12-24 12:28 +0200 http://bitbucket.org/pypy/pypy/changeset/8968b3201bcd/ Log: uncomment passing test diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -837,7 +837,6 @@ def test_unicode_keywords(self): - """ def f(**kwargs): assert kwargs["美"] == 42 f(**{"美" : 42}) @@ -845,7 +844,6 @@ def f(x): pass e = raises(TypeError, "f(**{'ü' : 19})") assert e.value.args[0] == "f() got an unexpected keyword argument 'ü'" - """ def test_starstarargs_dict_subclass(self): def f(**kwargs): From pypy.commits at gmail.com Mon Dec 24 10:28:21 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:21 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: improve exception text compliance, fix for non-ascii attribute Message-ID: <5c20fb15.1c69fb81.56fea.95fe@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95525:5766e9675e3e Date: 2018-12-24 14:03 +0200 http://bitbucket.org/pypy/pypy/changeset/5766e9675e3e/ Log: improve exception text compliance, fix for non-ascii attribute diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -36,29 +36,33 @@ return space.len(w_obj) -def checkattrname(space, w_name): +def checkattrname(space, w_name, msg): # This is a check to ensure that getattr/setattr/delattr only pass a - # string to the rest of the code. XXX not entirely sure if these three + # ascii string to the rest of the code. XXX not entirely sure if these # functions are the only way for non-string objects to reach # space.{get,set,del}attr()... - # Note that if w_name is already an exact string it must be returned - # unmodified (and not e.g. unwrapped-rewrapped). - if not space.is_w(space.type(w_name), space.w_text): - name = space.text_w(w_name) # typecheck - w_name = space.newtext(name) # rewrap as a real string + # Note that if w_name is already an exact string it must be ascii encoded + if not space.isinstance_w(w_name, space.w_text): + try: + name = space.text_w(w_name) # typecheck + except Exception as e: + raise oefmt(space.w_TypeError, + "%s(): attribute name must be string", msg) + if space.isinstance_w(w_name, space.w_unicode): + w_name = space.call_method(w_name, 'encode', space.newtext('ascii')) return w_name def delattr(space, w_object, w_name): """Delete a named attribute on an object. delattr(x, 'y') is equivalent to ``del x.y''.""" - w_name = checkattrname(space, w_name) + w_name = checkattrname(space, w_name, 'delattr') space.delattr(w_object, w_name) return space.w_None def getattr(space, w_object, w_name, w_defvalue=None): """Get a named attribute from an object. getattr(x, 'y') is equivalent to ``x.y''.""" - w_name = checkattrname(space, w_name) + w_name = checkattrname(space, w_name, 'getattr') try: return space.getattr(w_object, w_name) except OperationError as e: @@ -70,7 +74,7 @@ def hasattr(space, w_object, w_name): """Return whether the object has an attribute with the given name. (This is done by calling getattr(object, name) and catching exceptions.)""" - w_name = checkattrname(space, w_name) + w_name = checkattrname(space, w_name, 'hasattr') if space.findattr(w_object, w_name) is not None: return space.w_True else: @@ -216,7 +220,7 @@ def setattr(space, w_object, w_name, w_val): """Store a named attribute into an object. setattr(x, 'y', z) is equivalent to ``x.y = z''.""" - w_name = checkattrname(space, w_name) + w_name = checkattrname(space, w_name, 'setattr') space.setattr(w_object, w_name, w_val) return space.w_None diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -568,9 +568,9 @@ assert hasattr(x, 'bar') is False assert hasattr(x, 'abc') is False # CPython compliance assert hasattr(x, 'bac') is False # CPython compliance - raises(TypeError, hasattr, x, None) - raises(TypeError, hasattr, x, 42) - raises(UnicodeError, hasattr, x, u'\u5678') # cannot encode attr name + exc = raises(TypeError, hasattr, x, None) + assert str(exc.value) == 'hasattr(): attribute name must be string' + raises(UnicodeError, hasattr, x, u'\u5678') # 'ascii' codec can't encode def test_execfile_args(self): execfile(self.nullbytes) # works From pypy.commits at gmail.com Mon Dec 24 10:28:23 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:23 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: check for ascii field name Message-ID: <5c20fb17.1c69fb81.947a9.b03b@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95526:0d5a37575018 Date: 2018-12-24 14:30 +0200 http://bitbucket.org/pypy/pypy/changeset/0d5a37575018/ Log: check for ascii field name diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -14,7 +14,7 @@ from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers from pypy.module._rawffi.interp_rawffi import size_alignment from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr -from rpython.rlib import clibffi, rgc +from rpython.rlib import clibffi, rgc, rutf8 from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \ r_ulonglong from rpython.rtyper.lltypesystem import lltype, rffi @@ -163,6 +163,10 @@ if name in name_to_index: raise oefmt(space.w_ValueError, "duplicate field name %s", name) + try: + rutf8.check_ascii(name) + except rutf8.CheckError: + raise oefmt(space.w_TypeError, 'non-ascii field name') name_to_index[name] = i size, alignment, pos, bitsizes = size_alignment_pos( fields, is_union, pack) From pypy.commits at gmail.com Mon Dec 24 10:28:25 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:25 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: DecodeBuffer returns utf8 codepoints Message-ID: <5c20fb19.1c69fb81.49892.2d2e@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95527:74c350367634 Date: 2018-12-24 16:25 +0200 http://bitbucket.org/pypy/pypy/changeset/74c350367634/ Log: DecodeBuffer returns utf8 codepoints diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -302,35 +302,43 @@ def __init__(self, text=None): self.text = text self.pos = 0 + self.upos = 0 def set(self, space, w_decoded): check_decoded(space, w_decoded) self.text = space.utf8_w(w_decoded) self.pos = 0 + self.upos = 0 def reset(self): self.text = None self.pos = 0 + self.upos = 0 def get_chars(self, size): if self.text is None: return "" - available = len(self.text) - self.pos + lgt = codepoints_in_utf8(self.text) + available = lgt - self.upos if size < 0 or size > available: size = available assert size >= 0 if self.pos > 0 or size < available: start = self.pos - end = self.pos + size - assert start >= 0 - assert end >= 0 - chars = self.text[start:end] + ret = [] + pos = start + for i in range(size): + pos = next_codepoint_pos(self.text, pos) + self.upos += 1 + chars = self.text[start:pos] + self.pos = pos else: chars = self.text + self.pos = len(self.text) + self.upos = lgt - self.pos += size return chars def has_data(self): @@ -342,16 +350,18 @@ def next_char(self): if self.exhausted(): raise StopIteration - ch = self.text[self.pos] - self.pos = next_codepoint_pos(self.text, self.pos) + newpos = next_codepoint_pos(self.text, self.pos) + ch = self.text[self.pos:newpos] + self.pos = newpos + self.upos += 1 return ch def peek_char(self): # like next_char, but doesn't advance pos if self.exhausted(): raise StopIteration - ch = self.text[self.pos] - return ch + newpos = next_codepoint_pos(self.text, self.pos) + return self.text[self.pos:newpos] def find_newline_universal(self, limit): # Universal newline search. Find any of \r, \r\n, \n diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -1,6 +1,6 @@ import pytest try: - from hypothesis import given, strategies as st, settings + from hypothesis import given, strategies as st, settings, example except ImportError: pytest.skip("hypothesis required") import os @@ -63,6 +63,7 @@ assert buf.exhausted() @given(st.text(), st.lists(st.integers(min_value=0))) + at example(u'\x80', [1]) def test_readn_buffer(text, sizes): buf = DecodeBuffer(text.encode('utf-8')) strings = [] @@ -80,5 +81,5 @@ buf = DecodeBuffer(text.encode('utf-8')) for i in range(len(text)): ch = buf.next_char() - assert ch == text[i].encode('utf-8')[0] + assert ch == text[i].encode('utf-8') assert buf.exhausted() From pypy.commits at gmail.com Mon Dec 24 10:28:26 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 07:28:26 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix translation Message-ID: <5c20fb1a.1c69fb81.34317.b4a6@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95528:e55276d37c08 Date: 2018-12-24 17:27 +0200 http://bitbucket.org/pypy/pypy/changeset/e55276d37c08/ Log: fix translation diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -332,6 +332,8 @@ for i in range(size): pos = next_codepoint_pos(self.text, pos) self.upos += 1 + assert start >= 0 + assert pos >= 0 chars = self.text[start:pos] self.pos = pos else: @@ -351,7 +353,10 @@ if self.exhausted(): raise StopIteration newpos = next_codepoint_pos(self.text, self.pos) - ch = self.text[self.pos:newpos] + pos = self.pos + assert pos >= 0 + assert newpos >= 0 + ch = self.text[pos:newpos] self.pos = newpos self.upos += 1 return ch @@ -361,7 +366,10 @@ if self.exhausted(): raise StopIteration newpos = next_codepoint_pos(self.text, self.pos) - return self.text[self.pos:newpos] + pos = self.pos + assert pos >= 0 + assert newpos >= 0 + return self.text[pos:newpos] def find_newline_universal(self, limit): # Universal newline search. Find any of \r, \r\n, \n From pypy.commits at gmail.com Mon Dec 24 14:53:24 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 11:53:24 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix be8caf00c728 for bytes Message-ID: <5c213934.1c69fb81.dff1b.1362@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95529:7df64fa7f398 Date: 2018-12-24 21:02 +0200 http://bitbucket.org/pypy/pypy/changeset/7df64fa7f398/ Log: fix be8caf00c728 for bytes diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -545,7 +545,10 @@ # note: negative-based indexing from the end w_name = keyword_names_w[i - len(keywords)] except IndexError: - name = '?' + if keywords is None: + name = '?' + else: + name = keywords[i] else: w_enc = space.newtext(space.sys.defaultencoding) w_err = space.newtext("replace") diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -54,6 +54,9 @@ pass class DummySpace(object): + class sys: + defaultencoding = 'utf-8' + def newtuple(self, items): return tuple(items) From pypy.commits at gmail.com Mon Dec 24 14:53:26 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 11:53:26 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix 74c350367634 for find_crlf Message-ID: <5c213936.1c69fb81.6be62.fb58@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95530:3fb6c618af46 Date: 2018-12-24 21:37 +0200 http://bitbucket.org/pypy/pypy/changeset/3fb6c618af46/ Log: fix 74c350367634 for find_crlf diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -419,6 +419,7 @@ except StopIteration: # This is the tricky case: we found a \r right at the end self.pos -= 1 + self.upos -= 1 return False return False From pypy.commits at gmail.com Mon Dec 24 14:53:28 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 11:53:28 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: change unicode_w(o) to utf8_w(o).decode('utf8') Message-ID: <5c213938.1c69fb81.10ea8.2f91@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95531:932573eb4bec Date: 2018-12-24 21:52 +0200 http://bitbucket.org/pypy/pypy/changeset/932573eb4bec/ Log: change unicode_w(o) to utf8_w(o).decode('utf8') diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -422,8 +422,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) - assert space.unicode_w(PyUnicode_FromEncodedObject( - space, space.wrap(s_text), null_charp, None)) == u_text + assert space.utf8_w(PyUnicode_FromEncodedObject(space, + space.wrap(s_text), null_charp, None)).decode('utf8') == u_text rffi.free_charp(b_text) def test_mbcs(self, space): From pypy.commits at gmail.com Tue Dec 25 01:46:46 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 22:46:46 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: str(unicode) encodes with ascii Message-ID: <5c21d256.1c69fb81.388ac.8305@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95532:93f739e28ea2 Date: 2018-12-25 08:02 +0200 http://bitbucket.org/pypy/pypy/changeset/93f739e28ea2/ Log: str(unicode) encodes with ascii diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -230,7 +230,7 @@ return space.newtext(_repr_function(self._utf8)) def descr_str(self, space): - return encode_object(space, self, 'utf8', 'strict') + return encode_object(space, self, 'ascii', 'strict') def descr_hash(self, space): x = compute_hash(self._utf8) From pypy.commits at gmail.com Tue Dec 25 01:48:55 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Dec 2018 22:48:55 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: document this branch Message-ID: <5c21d2d7.1c69fb81.189e0.b5ce@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95533:fef23fda84d1 Date: 2018-12-25 08:48 +0200 http://bitbucket.org/pypy/pypy/changeset/fef23fda84d1/ Log: document this branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -64,3 +64,7 @@ .. branch: cleanup-test_lib_pypy Update most test_lib_pypy/ tests and move them to extra_tests/. + +.. branch: unicode-utf8 + +Use utf8 internally to represent unicode From pypy.commits at gmail.com Tue Dec 25 09:33:44 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 25 Dec 2018 06:33:44 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #392 Message-ID: <5c223fc8.1c69fb81.b0577.4a44@mx.google.com> Author: Armin Rigo Branch: Changeset: r3177:126a323eb964 Date: 2018-12-25 15:33 +0100 http://bitbucket.org/cffi/cffi/changeset/126a323eb964/ Log: Issue #392 Workaround for a pycparser issue. diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -16,6 +16,13 @@ except ImportError: lock = None +def _workaround_for_static_import_finders(): + # Issue #392: packaging tools like cx_Freeze can not find these + # because pycparser uses exec dynamic import. This is an obscure + # workaround. This function is never called. + import pycparser.yacctab + import pycparser.lextab + CDEF_SOURCE_STRING = "" _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", re.DOTALL | re.MULTILINE) From pypy.commits at gmail.com Fri Dec 28 12:18:28 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 28 Dec 2018 09:18:28 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: add myself Message-ID: <5c265ae4.1c69fb81.91bec.bc43@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5929:56f50e784f02 Date: 2018-12-28 18:18 +0100 http://bitbucket.org/pypy/extradoc/changeset/56f50e784f02/ Log: add myself diff --git a/sprintinfo/ddorf2019/people.txt b/sprintinfo/ddorf2019/people.txt --- a/sprintinfo/ddorf2019/people.txt +++ b/sprintinfo/ddorf2019/people.txt @@ -11,6 +11,7 @@ Carl Friedrich Bolz-Tereick always there private Matti Picus Feb 4? - 9? any suggestions?? Manuel? Feb 4 - 7 share a room? +Antonio Cuni Feb 3 - 9 airbnb ============================ ============== ===================== From pypy.commits at gmail.com Sat Dec 29 15:38:03 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 29 Dec 2018 12:38:03 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: add andrew lawrence Message-ID: <5c27db2b.1c69fb81.0d87.009f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r5930:4e7d0c4adb4a Date: 2018-12-29 20:37 +0000 http://bitbucket.org/pypy/extradoc/changeset/4e7d0c4adb4a/ Log: add andrew lawrence diff --git a/sprintinfo/ddorf2019/people.txt b/sprintinfo/ddorf2019/people.txt --- a/sprintinfo/ddorf2019/people.txt +++ b/sprintinfo/ddorf2019/people.txt @@ -5,14 +5,15 @@ column are known to be coming but there are no details available yet from them. -============================ ============== ===================== +============================ ============== ====================== Name Arrive/Depart Accomodation -============================ ============== ===================== +============================ ============== ====================== Carl Friedrich Bolz-Tereick always there private Matti Picus Feb 4? - 9? any suggestions?? Manuel? Feb 4 - 7 share a room? Antonio Cuni Feb 3 - 9 airbnb -============================ ============== ===================== +Andrew Lawrence Feb 3 - 9 backpackers Düsseldorf +============================ ============== ====================== From pypy.commits at gmail.com Sat Dec 29 21:22:08 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 29 Dec 2018 18:22:08 -0800 (PST) Subject: [pypy-commit] pypy default: unicode(encoding='x') returns the empty string u''. Message-ID: <5c282bd0.1c69fb81.d8910.0092@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r95534:24f6622d7e24 Date: 2018-12-29 23:06 +0100 http://bitbucket.org/pypy/pypy/changeset/24f6622d7e24/ Log: unicode(encoding='x') returns the empty string u''. This behavior is explicitly tested in py3.5. diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -1163,9 +1163,8 @@ assert type(unicode(z)) is unicode assert unicode(z) == u'foobaz' # - # two completely corner cases where we differ from CPython: - #assert unicode(encoding='supposedly_the_encoding') == u'' - #assert unicode(errors='supposedly_the_error') == u'' + assert unicode(encoding='supposedly_the_encoding') == u'' + assert unicode(errors='supposedly_the_error') == u'' e = raises(TypeError, unicode, u'', 'supposedly_the_encoding') assert str(e.value) == 'decoding Unicode is not supported' e = raises(TypeError, unicode, u'', errors='supposedly_the_error') diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -192,23 +192,20 @@ return space.newlist_unicode(lst) @staticmethod - @unwrap_spec(w_string=WrappedDefault("")) - def descr_new(space, w_unicodetype, w_string, w_encoding=None, + def descr_new(space, w_unicodetype, w_string=None, w_encoding=None, w_errors=None): - # NB. the default value of w_obj is really a *wrapped* empty string: - # there is gateway magic at work - w_obj = w_string - encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - if encoding is None and errors is None: - # this is very quick if w_obj is already a w_unicode - w_value = unicode_from_object(space, w_obj) + if w_string is None: + w_value = W_UnicodeObject.EMPTY + elif encoding is None and errors is None: + # this is very quick if w_string is already a w_unicode + w_value = unicode_from_object(space, w_string) else: - if space.isinstance_w(w_obj, space.w_unicode): + if space.isinstance_w(w_string, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_value = unicode_from_encoded_object(space, w_obj, + w_value = unicode_from_encoded_object(space, w_string, encoding, errors) if space.is_w(w_unicodetype, space.w_unicode): return w_value From pypy.commits at gmail.com Sat Dec 29 21:22:10 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 29 Dec 2018 18:22:10 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5c282bd2.1c69fb81.ee85b.37b6@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.5 Changeset: r95535:f81df27a6bf7 Date: 2018-12-30 03:20 +0100 http://bitbucket.org/pypy/pypy/changeset/f81df27a6bf7/ Log: hg merge default diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -1105,9 +1105,8 @@ assert type(str(z)) is str assert str(z) == u'foobaz' # - # two completely corner cases where we differ from CPython: - #assert unicode(encoding='supposedly_the_encoding') == u'' - #assert unicode(errors='supposedly_the_error') == u'' + assert str(encoding='supposedly_the_encoding') == u'' + assert str(errors='supposedly_the_error') == u'' e = raises(TypeError, str, u'', 'supposedly_the_encoding') assert str(e.value) == 'decoding str is not supported' e = raises(TypeError, str, u'', errors='supposedly_the_error') diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -223,20 +223,18 @@ @staticmethod def descr_new(space, w_unicodetype, w_object=None, w_encoding=None, w_errors=None): - if w_object is None: - w_object = W_UnicodeObject.EMPTY - w_obj = w_object - encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - if encoding is None and errors is None: - # this is very quick if w_obj is already a w_unicode - w_value = unicode_from_object(space, w_obj) + if w_object is None: + w_value = W_UnicodeObject.EMPTY + elif encoding is None and errors is None: + # this is very quick if w_object is already a w_unicode + w_value = unicode_from_object(space, w_object) else: - if space.isinstance_w(w_obj, space.w_unicode): + if space.isinstance_w(w_object, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - w_value = unicode_from_encoded_object(space, w_obj, + w_value = unicode_from_encoded_object(space, w_object, encoding, errors) if space.is_w(w_unicodetype, space.w_unicode): return w_value From pypy.commits at gmail.com Sat Dec 29 21:42:21 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 29 Dec 2018 18:42:21 -0800 (PST) Subject: [pypy-commit] pypy py3.6: CPython Issue 18550: Check error values in socket.setblocking() Message-ID: <5c28308d.1c69fb81.fa0a6.f8c9@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r95536:cafe126a986a Date: 2018-08-06 15:49 +0200 http://bitbucket.org/pypy/pypy/changeset/cafe126a986a/ Log: CPython Issue 18550: Check error values in socket.setblocking() diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -608,14 +608,17 @@ return space.newint(count) @unwrap_spec(flag=int) - def setblocking_w(self, flag): + def setblocking_w(self, space, flag): """setblocking(flag) Set the socket to blocking (flag is true) or non-blocking (false). setblocking(True) is equivalent to settimeout(None); setblocking(False) is equivalent to settimeout(0.0). """ - self.sock.setblocking(bool(flag)) + try: + self.sock.setblocking(bool(flag)) + except SocketError as e: + raise converted_error(space, e) @unwrap_spec(level=int, optname=int) def setsockopt_w(self, space, level, optname, w_optval): @@ -654,7 +657,10 @@ timeout = space.float_w(w_timeout) if timeout < 0.0: raise oefmt(space.w_ValueError, "Timeout value out of range") - self.sock.settimeout(timeout) + try: + self.sock.settimeout(timeout) + except SocketError as e: + raise converted_error(space, e) @unwrap_spec(nbytes=int, flags=int) def recv_into_w(self, space, w_buffer, nbytes=0, flags=0): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -402,6 +402,12 @@ e = raises(OSError, s.close) assert e.value.errno in (errno.EBADF, errno.ENOTSOCK) + def test_setblocking_invalidfd(self): + import errno, _socket + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) + _socket.socket(fileno=s.fileno()).close() + raises(OSError, s.setblocking, False) + def test_socket_connect(self): import _socket, os s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1270,7 +1270,8 @@ getprotobyname = external('getprotobyname', [rffi.CCHARP], lltype.Ptr(cConfig.protoent)) if _POSIX: - fcntl = external('fcntl', [socketfd_type, rffi.INT, rffi.INT], rffi.INT) + fcntl = external('fcntl', [socketfd_type, rffi.INT, rffi.INT], rffi.INT, + save_err=SAVE_ERR) socketpair_t = rffi.CArray(socketfd_type) socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(socketpair_t)], rffi.INT, @@ -1282,7 +1283,7 @@ if _WIN32: ioctlsocket = external('ioctlsocket', [socketfd_type, rffi.LONG, rffi.ULONGP], - rffi.INT) + rffi.INT, save_err=SAVE_ERR) select = external('select', [rffi.INT, fd_set, fd_set, diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -567,17 +567,21 @@ if hasattr(_c, 'fcntl'): def _setblocking(self, block): orig_delay_flag = intmask(_c.fcntl(self.fd, _c.F_GETFL, 0)) + if orig_delay_flag == -1: + raise self.error_handler() if block: delay_flag = orig_delay_flag & ~_c.O_NONBLOCK else: delay_flag = orig_delay_flag | _c.O_NONBLOCK if orig_delay_flag != delay_flag: - _c.fcntl(self.fd, _c.F_SETFL, delay_flag) + if _c.fcntl(self.fd, _c.F_SETFL, delay_flag) == -1: + raise self.error_handler() elif hasattr(_c, 'ioctlsocket'): def _setblocking(self, block): flag = lltype.malloc(rffi.ULONGP.TO, 1, flavor='raw') flag[0] = rffi.cast(rffi.ULONG, not block) - _c.ioctlsocket(self.fd, _c.FIONBIO, flag) + if _c.ioctlsocket(self.fd, _c.FIONBIO, flag) != 0: + raise self.error_handler() lltype.free(flag, flavor='raw') if hasattr(_c, 'poll') and not _c.poll_may_be_broken: From pypy.commits at gmail.com Sat Dec 29 21:42:23 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 29 Dec 2018 18:42:23 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Port CPython commit 91108f0: Issue #25210: Change error message of do_richcompare() Message-ID: <5c28308f.1c69fb81.50c42.6c97@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r95537:0f59af55378c Date: 2018-08-05 17:33 +0200 http://bitbucket.org/pypy/pypy/changeset/0f59af55378c/ Log: Port CPython commit 91108f0: Issue #25210: Change error message of do_richcompare() diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -619,7 +619,8 @@ # # if we arrived here, they are unorderable raise oefmt(space.w_TypeError, - "unorderable types: %T %s %T", w_obj1, symbol, w_obj2) + "'%s' not supported between instances of '%T' and '%T'", + symbol, w_obj1, w_obj2) return func_with_new_name(comparison_impl, 'comparison_%s_impl'%left.strip('_')) From pypy.commits at gmail.com Sat Dec 29 21:42:24 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 29 Dec 2018 18:42:24 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Fix some doctests: PyPy does not add the function name to the error message, Message-ID: <5c283090.1c69fb81.22113.fc2d@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r95538:d510a67badcd Date: 2018-12-30 03:41 +0100 http://bitbucket.org/pypy/pypy/changeset/d510a67badcd/ Log: Fix some doctests: PyPy does not add the function name to the error message, and one exception was wrong. diff --git a/lib-python/3/test/test_unpack_ex.py b/lib-python/3/test/test_unpack_ex.py --- a/lib-python/3/test/test_unpack_ex.py +++ b/lib-python/3/test/test_unpack_ex.py @@ -244,16 +244,16 @@ ... TypeError: ...got multiple values for keyword argument 'x' - >>> f(x=5, **{'x': 3}, **{'x': 2}) - Traceback (most recent call last): - ... - TypeError: f() got multiple values for keyword argument 'x' - - >>> f(**{1: 3}, **{1: 5}) + >>> f(x=5, **{'x': 3}, **{'x': 2}) # doctest:+ELLIPSIS Traceback (most recent call last): ... TypeError: ...got multiple values for keyword argument 'x' + >>> f(**{1: 3}, **{1: 5}) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + TypeError: ...keywords must be strings... + >>> f(x=5, **{'x': 3}, **{'x': 2}) # doctest:+ELLIPSIS Traceback (most recent call last): ... From pypy.commits at gmail.com Sun Dec 30 08:58:38 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 30 Dec 2018 05:58:38 -0800 (PST) Subject: [pypy-commit] pypy py3.6: The variable annotation: '(var): int' should consider (var) as an expression, Message-ID: <5c28cf0e.1c69fb81.c3e75.bd48@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r95539:26c82f5c208f Date: 2018-12-30 13:17 +0100 http://bitbucket.org/pypy/pypy/changeset/26c82f5c208f/ Log: The variable annotation: '(var): int' should consider (var) as an expression, and not cause var to become a local variable. diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -429,11 +429,15 @@ self.scope.contains_annotated = True target = assign.target if isinstance(target, ast.Name): - scope = SYM_ANNOTATED + # XXX Should check (and fail) for previous 'global' annotation. name = target.id + scope = SYM_BLANK + if assign.simple: + scope |= SYM_ANNOTATED if assign.value: scope |= SYM_USED - self.note_symbol(name, scope) + if scope: + self.note_symbol(name, scope) else: target.walkabout(self) if assign.value is not None: diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -517,6 +517,9 @@ fscp = self.func_scope("def f(): implicit_global[0]: int") assert fscp.lookup("implicit_global") == symtable.SCOPE_GLOBAL_IMPLICIT + fscp = self.func_scope("def f(): (implicit_global): int") + assert fscp.lookup("implicit_global") == symtable.SCOPE_UNKNOWN + def test_issue13343(self): scp = self.mod_scope("lambda *, k1=x, k2: None") assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py --- a/pypy/interpreter/test/test_annotations.py +++ b/pypy/interpreter/test/test_annotations.py @@ -73,6 +73,15 @@ ... ''' + def test_non_simple_func_annotation(self): + ''' + a = 5 + def f(): + (a): int + return a + assert f() == 5 + ''' + def test_repeated_setup(self): # each exec will run another SETUP_ANNOTATIONS # we want to confirm that this doesn't blow away @@ -147,4 +156,3 @@ """ c = compile(s, "f", "exec") assert c.co_firstlineno == 3 - From pypy.commits at gmail.com Sun Dec 30 08:58:40 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 30 Dec 2018 05:58:40 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Check conflict between 'global' and variable annotation. Message-ID: <5c28cf10.1c69fb81.d7951.8b78@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r95540:3f85fe07a914 Date: 2018-12-30 14:57 +0100 http://bitbucket.org/pypy/pypy/changeset/3f85fe07a914/ Log: Check conflict between 'global' and variable annotation. This fixes the last failure in test_grammar. diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -429,8 +429,17 @@ self.scope.contains_annotated = True target = assign.target if isinstance(target, ast.Name): - # XXX Should check (and fail) for previous 'global' annotation. name = target.id + old_role = self.scope.lookup_role(name) + if assign.simple: + if old_role & SYM_GLOBAL: + raise SyntaxError( + "annotated name '%s' can't be global" % name, + assign.lineno, assign.col_offset) + if old_role & SYM_NONLOCAL: + raise SyntaxError( + "annotated name '%s' can't be nonlocal" % name, + assign.lineno, assign.col_offset) scope = SYM_BLANK if assign.simple: scope |= SYM_ANNOTATED diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -494,6 +494,14 @@ assert exc_global.msg == "annotated name 'x' can't be global" assert exc_global.lineno == 3 + def test_annotation_global2(self): + src_global = ("def f():\n" + " global x\n" + " x: int\n") + exc_global = py.test.raises(SyntaxError, self.func_scope, src_global).value + assert exc_global.msg == "annotated name 'x' can't be global" + assert exc_global.lineno == 3 + def test_annotation_nonlocal(self): src_nonlocal = ("def f():\n" " x: int\n"