From pypy.commits at gmail.com Thu Nov 1 01:13:54 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 31 Oct 2018 22:13:54 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: typo, repeat of 10fdcbcd9d3f Message-ID: <5bda8b92.1c69fb81.ebe3f.7462@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95262:92689c7f2938 Date: 2018-10-30 13:41 +0200 http://bitbucket.org/pypy/pypy/changeset/92689c7f2938/ Log: typo, repeat of 10fdcbcd9d3f diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -77,7 +77,7 @@ def fsdecode(space, w_string): from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) - errorhandler=state.decode_error_handler, + errorhandler=state.decode_error_handler if _WIN32: bytes = space.bytes_w(w_string) uni = str_decode_mbcs(bytes, 'strict', True, errorhandler, From pypy.commits at gmail.com Thu Nov 1 01:13:57 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 31 Oct 2018 22:13:57 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: convert File{En, De}codingObject to unicode via traits.as_str0, fix translation Message-ID: <5bda8b95.1c69fb81.5282d.8a1a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95263:78258e983d74 Date: 2018-10-31 23:14 +0200 http://bitbucket.org/pypy/pypy/changeset/78258e983d74/ Log: convert File{En,De}codingObject to unicode via traits.as_str0, fix translation diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -80,13 +80,15 @@ errorhandler=state.decode_error_handler if _WIN32: bytes = space.bytes_w(w_string) - uni = str_decode_mbcs(bytes, 'strict', True, errorhandler, - force_ignore=False)[0] + slen = len(bytes) + uni, size = runicode.str_decode_mbcs(bytes, slen, 'strict', final=True, + errorhandler=errorhandler, force_ignore=False) elif _MACOSX: bytes = space.bytes_w(w_string) - uni = str_decode_utf8( + utf8 = str_decode_utf8( bytes, 'surrogateescape', final=True, allow_surrogates=False)[0] + uni = space.realunicode_w(utf8) elif space.sys.filesystemencoding is None or state.codec_need_encodings: # bootstrap check: if the filesystemencoding isn't initialized # or the filesystem codec is implemented in Python we cannot @@ -101,6 +103,7 @@ return space.call_method(w_string, 'decode', getfilesystemencoding(space), space.newtext('surrogateescape')) + assert isinstance(uni, unicode) return space.newtext(runicode.unicode_encode_utf_8(uni, len(uni), 'strict', allow_surrogates=True), len(uni)) diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py --- a/pypy/module/_cffi_backend/errorbox.py +++ b/pypy/module/_cffi_backend/errorbox.py @@ -89,7 +89,7 @@ return w_text = self.space.call_function(w_done) - p = rffi.unicode2wcharp(self.space.utf8_w(w_text), + p = rffi.unicode2wcharp(self.space.realunicode_w(w_text), track_allocation=False) if self.text_p: rffi.free_wcharp(self.text_p, track_allocation=False) diff --git a/pypy/module/posix/interp_nt.py b/pypy/module/posix/interp_nt.py --- a/pypy/module/posix/interp_nt.py +++ b/pypy/module/posix/interp_nt.py @@ -96,7 +96,7 @@ raise LLNotImplemented("GetFinalPathNameByHandle not available on " "this platform") - hFile = win32traits.CreateFile(path, 0, 0, None, + hFile = win32traits.CreateFile(traits.as_str0(path), 0, 0, None, win32traits.OPEN_EXISTING, win32traits.FILE_FLAG_BACKUP_SEMANTICS, rwin32.NULL_HANDLE) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -56,7 +56,10 @@ return self.space.fsencode_w(self.w_obj) def as_unicode(self): - return self.space.utf8_w(self.w_obj).decode('utf8') + ret = self.space.realunicode_w(self.w_obj) + if u'\x00' in ret: + raise oefmt(self.space.w_ValueError, "embedded null character") + return ret class FileDecoder(object): is_unicode = False @@ -69,7 +72,10 @@ return self.space.bytesbuf0_w(self.w_obj) def as_unicode(self): - return self.space.fsdecode_w(self.w_obj) + ret = self.space.fsdecode_w(self.w_obj) + if u'\x00' in ret: + raise oefmt(self.space.w_ValueError, "embedded null character") + return ret @specialize.memo() def make_dispatch_function(func, tag, allow_fd_fn=None): @@ -889,7 +895,7 @@ # started through main() instead of wmain() rwin32._wgetenv(u"") for key, value in rwin32._wenviron_items(): - space.setitem(w_env, space.newtext(key), space.newunicode(value)) + space.setitem(w_env, space.newtext(key), space.newtext(value)) @unwrap_spec(name=unicode, value=unicode) def putenv(space, name, value): diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py --- a/rpython/rlib/_os_support.py +++ b/rpython/rlib/_os_support.py @@ -64,6 +64,8 @@ assert path is not None if isinstance(path, unicode): return path + elif isinstance(path, str): + raise RuntimeError('str given where unicode expected') else: return path.as_unicode() diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,10 +137,7 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', - 'RPY_EXTERN void* enter_suppress_iph();', - 'RPY_EXTERN void exit_suppress_iph(void* handle);', - ] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: separate_module_sources = [] post_include_bits = [] @@ -238,8 +235,7 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', - 'direct.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -685,8 +685,7 @@ # XXX 'traverse' is ignored, and everything related to # the "reparse points" is missing win32traits = make_win32_traits(traits) - - hFile = win32traits.CreateFile(path, + hFile = win32traits.CreateFile(traits.as_str0(path), win32traits.FILE_READ_ATTRIBUTES, 0, lltype.nullptr(rwin32.LPSECURITY_ATTRIBUTES.TO), From pypy.commits at gmail.com Thu Nov 1 01:13:59 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 31 Oct 2018 22:13:59 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: str -> utf_w, unicode -> realunicode_w Message-ID: <5bda8b97.1c69fb81.c3491.73d5@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95264:491e2a6351bd Date: 2018-11-01 07:10 +0200 http://bitbucket.org/pypy/pypy/changeset/491e2a6351bd/ Log: str -> utf_w, unicode -> realunicode_w diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -9,4 +9,5 @@ * revisit all places where we do utf8.decode('utf-8'), they should work directly with utf8 - rutf8.utf8_encode_mbcs - unicodehelper.fsencode - - interp_posix.FileEncoder.as_unicode (used in win32) +* convert all realunicode_w to unicode_w after we flush out all old uses of unicode_w +* benchmark diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -576,8 +576,10 @@ assert typ in (int, str, float, unicode, r_longlong, r_uint, r_ulonglong, bool) if typ is r_int is r_longlong: return 'gateway_r_longlong_w' - elif typ in (str, unicode): - return typ.__name__ + '_w' + elif typ is str: + return 'utf8_w' + elif typ is unicode: + return 'realunicode_w' elif typ is bool: # For argument clinic's "bool" specifier: accept any object, and # convert it to a boolean value. If you don't want this From pypy.commits at gmail.com Thu Nov 1 01:14:01 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 31 Oct 2018 22:14:01 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: one more place that needs to use unicode Message-ID: <5bda8b99.1c69fb81.70a4a.11bd@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95265:16c587daa068 Date: 2018-11-01 07:11 +0200 http://bitbucket.org/pypy/pypy/changeset/16c587daa068/ Log: one more place that needs to use unicode diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py --- a/pypy/module/posix/interp_scandir.py +++ b/pypy/module/posix/interp_scandir.py @@ -27,7 +27,7 @@ if space.isinstance_w(w_path, space.w_bytes): raise oefmt(space.w_TypeError, "os.scandir() doesn't support bytes path" " on Windows, use Unicode instead") - path = space.utf8_w(w_path) + path = space.realunicode_w(w_path) result_is_bytes = False # 'path' is always bytes on posix and always unicode on windows From pypy.commits at gmail.com Thu Nov 1 01:14:03 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 31 Oct 2018 22:14:03 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: add asserts for translation - why is this needed? Message-ID: <5bda8b9b.1c69fb81.cdf83.f2ec@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95266:692cf928c31b Date: 2018-11-01 07:12 +0200 http://bitbucket.org/pypy/pypy/changeset/692cf928c31b/ Log: add asserts for translation - why is this needed? diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -9,5 +9,6 @@ * revisit all places where we do utf8.decode('utf-8'), they should work directly with utf8 - rutf8.utf8_encode_mbcs - unicodehelper.fsencode +* remove asserts from _WIN32 paths in rlib.rposix.re{name,place} * convert all realunicode_w to unicode_w after we flush out all old uses of unicode_w * benchmark diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -1269,6 +1269,8 @@ win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) + assert isinstance(path1, unicode) + assert isinstance(path2, unicode) if not win32traits.MoveFileEx(path1, path2, 0): raise rwin32.lastSavedWindowsError() @@ -1279,6 +1281,8 @@ win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) + assert isinstance(path1, unicode) + assert isinstance(path2, unicode) ret = win32traits.MoveFileEx(path1, path2, win32traits.MOVEFILE_REPLACE_EXISTING) if not ret: From pypy.commits at gmail.com Thu Nov 1 05:34:33 2018 From: pypy.commits at gmail.com (stian) Date: Thu, 01 Nov 2018 02:34:33 -0700 (PDT) Subject: [pypy-commit] pypy math-improvements: Simplefy code Message-ID: <5bdac8a9.1c69fb81.b157c.8366@mx.google.com> Author: stian Branch: math-improvements Changeset: r95267:e7232ea18ab2 Date: 2018-11-01 10:33 +0100 http://bitbucket.org/pypy/pypy/changeset/e7232ea18ab2/ Log: Simplefy code diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -588,29 +588,7 @@ # Fallback to Long. return self.lt(rbigint.fromint(other)) - osign = 1 - if other == 0: - osign = 0 - elif other < 0: - osign = -1 - - if self.sign > osign: - return False - elif self.sign < osign: - return True - - digits = self.numdigits() - - if digits > 1: - if osign == 1: - return False - else: - return True - - d1 = self.sign * self.digit(0) - if d1 < other: - return True - return False + return _x_int_lt(self, other, False) def le(self, other): return not other.lt(self) @@ -622,29 +600,7 @@ # Fallback to Long. return self.lt(rbigint.fromint(other)) - osign = 1 - if other == 0: - osign = 0 - elif other < 0: - osign = -1 - - if self.sign > osign: - return False - elif self.sign < osign: - return True - - digits = self.numdigits() - - if digits > 1: - if osign == 1: - return False - else: - return True - - d1 = self.sign * self.digit(0) - if d1 <= other: - return True - return False + return _x_int_lt(self, other, True) def gt(self, other): return other.lt(self) @@ -2186,6 +2142,36 @@ rem.sign = - rem.sign return z, rem +def _x_int_lt(a, b, eq=False): + """ Compare bigint a with int b for less than or less than or equal """ + osign = 1 + if b == 0: + osign = 0 + elif b < 0: + osign = -1 + + if a.sign > osign: + return False + elif a.sign < osign: + return True + + digits = a.numdigits() + + if digits > 1: + if osign == 1: + return False + else: + return True + + d1 = a.sign * a.digit(0) + if eq: + if d1 <= b: + return True + else: + if d1 < b: + return True + return False + # ______________ conversions to double _______________ def _AsScaledDouble(v): From pypy.commits at gmail.com Thu Nov 1 12:19:29 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Nov 2018 09:19:29 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: avoid IndexError in codepoint_at_pos Message-ID: <5bdb2791.1c69fb81.1cbbc.2c6a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95268:e625965aa9cc Date: 2018-11-01 18:17 +0200 http://bitbucket.org/pypy/pypy/changeset/e625965aa9cc/ Log: avoid IndexError in codepoint_at_pos diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -155,18 +155,19 @@ def codepoint_at_pos(code, pos): """ Give a codepoint in code at pos - assumes valid utf8, no checking! """ + lgt = len(code) ordch1 = ord(code[pos]) - if ordch1 <= 0x7F: + if ordch1 <= 0x7F or pos +1 >= lgt: return ordch1 ordch2 = ord(code[pos+1]) - if ordch1 <= 0xDF: + if ordch1 <= 0xDF or pos +2 >= lgt: # 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz return (ordch1 << 6) + ordch2 - ( (0xC0 << 6) + 0x80 ) ordch3 = ord(code[pos+2]) - if ordch1 <= 0xEF: + if ordch1 <= 0xEF or pos + 3 >= lgt: # 1110xxxx 10yyyyyy 10zzzzzz -> 00000000 xxxxyyyy yyzzzzzz return (ordch1 << 12) + (ordch2 << 6) + ordch3 - ( (0xE0 << 12) + (0x80 << 6) + 0x80 ) From pypy.commits at gmail.com Thu Nov 1 12:19:31 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Nov 2018 09:19:31 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: do not re-encode in surrogateescape (is this correct?) Message-ID: <5bdb2793.1c69fb81.5d3b5.98aa@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95269:db15a8ce07d7 Date: 2018-11-01 18:18 +0200 http://bitbucket.org/pypy/pypy/changeset/db15a8ce07d7/ Log: do not re-encode in surrogateescape (is this correct?) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1224,6 +1224,9 @@ # surrogatepass? break pos = _pos + if errors == 'surrogateescape': + #escape + return space.newbytes(utf8) w_object = space.newtext(utf8) if errors is None or errors == 'strict': if encoding is None or encoding == 'utf-8': From pypy.commits at gmail.com Fri Nov 2 03:34:23 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 02 Nov 2018 00:34:23 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix bad merge from fix-sre-problems Message-ID: <5bdbfdff.1c69fb81.e2238.6e1f@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95270:aac35506e1a4 Date: 2018-11-02 09:17 +0200 http://bitbucket.org/pypy/pypy/changeset/aac35506e1a4/ Log: fix bad merge from fix-sre-problems diff --git a/rpython/rlib/rsre/test/support.py b/rpython/rlib/rsre/test/support.py --- a/rpython/rlib/rsre/test/support.py +++ b/rpython/rlib/rsre/test/support.py @@ -111,9 +111,9 @@ start, end = _adjust(start, end, len(string)) start = Position(start) end = Position(end) - ctx = MatchContextForTests(pattern, string, start, end, flags) + ctx = MatchContextForTests(string, start, end, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None @@ -125,8 +125,8 @@ start, end = _adjust(start, end, len(string)) start = Position(start) end = Position(end) - ctx = MatchContextForTests(pattern, string, start, end, flags) - if search_context(ctx): + ctx = MatchContextForTests(string, start, end, flags) + if search_context(ctx, pattern): return ctx else: return None From pypy.commits at gmail.com Sun Nov 4 07:46:07 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Nov 2018 04:46:07 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix merge from fix-sre-problems Message-ID: <5bdeea0f.1c69fb81.c4116.b5da@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95271:ac5e084fa5b9 Date: 2018-11-04 06:47 -0500 http://bitbucket.org/pypy/pypy/changeset/ac5e084fa5b9/ Log: fix merge from fix-sre-problems diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -460,7 +460,7 @@ ptr = self.start_ptr if not self.next_char_ok(ctx, pattern, ptr, self.ppos3): return - self.start_ptr = ptr + 1 + self.start_ptr = ctx.next(ptr) return self.find_first_result(ctx, pattern) def next_char_ok(self, ctx, pattern, ptr, ppos): @@ -736,9 +736,9 @@ startptr, length_bytes = get_group_ref(ctx, marks, pattern.pat(ppos)) if length_bytes < 0: return # group was not previously defined - if not match_repeated_ignore(ctx, ptr, startptr, length_bytes): + ptr = match_repeated_ignore(ctx, ptr, startptr, length_bytes) + if ptr < ctx.ZERO: return # no match - ptr = ctx.go_forward_by_bytes(ptr, length_bytes) ppos += 1 elif op == OPCODE_GROUPREF_EXISTS: diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -66,12 +66,12 @@ return position_high - position_low -def make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags): +def make_utf8_ctx(utf8string, bytestart, byteend, flags): if bytestart < 0: bytestart = 0 elif bytestart > len(utf8string): bytestart = len(utf8string) if byteend < 0: byteend = 0 elif byteend > len(utf8string): byteend = len(utf8string) - ctx = Utf8MatchContext(pattern, utf8string, bytestart, byteend, flags) + ctx = Utf8MatchContext(utf8string, bytestart, byteend, flags) ctx.debug_check_pos(bytestart) ctx.debug_check_pos(byteend) return ctx @@ -81,8 +81,8 @@ # utf8string. from rpython.rlib.rsre.rsre_core import search_context - ctx = make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags) - if search_context(ctx): + ctx = make_utf8_ctx(utf8string, bytestart, byteend, flags) + if search_context(ctx, pattern): return ctx else: return None @@ -93,9 +93,9 @@ # utf8string. from rpython.rlib.rsre.rsre_core import match_context - ctx = make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags) + ctx = make_utf8_ctx(utf8string, bytestart, byteend, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None diff --git a/rpython/rlib/rsre/test/support.py b/rpython/rlib/rsre/test/support.py --- a/rpython/rlib/rsre/test/support.py +++ b/rpython/rlib/rsre/test/support.py @@ -111,9 +111,9 @@ start, end = _adjust(start, end, len(string)) start = Position(start) end = Position(end) - ctx = MatchContextForTests(pattern, string, start, end, flags) + ctx = MatchContextForTests(string, start, end, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None @@ -125,8 +125,8 @@ start, end = _adjust(start, end, len(string)) start = Position(start) end = Position(end) - ctx = MatchContextForTests(pattern, string, start, end, flags) - if search_context(ctx): + ctx = MatchContextForTests(string, start, end, flags) + if search_context(ctx, pattern): return ctx else: return None From pypy.commits at gmail.com Sun Nov 4 07:46:10 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Nov 2018 04:46:10 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge unicode-utf8 into branch Message-ID: <5bdeea12.1c69fb81.c21ad.35c3@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95272:f2ad44a9d50b Date: 2018-11-04 07:25 -0500 http://bitbucket.org/pypy/pypy/changeset/f2ad44a9d50b/ Log: merge unicode-utf8 into branch diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c --- a/pypy/module/cpyext/test/buffer_test.c +++ b/pypy/module/cpyext/test/buffer_test.c @@ -192,6 +192,10 @@ */ #define GET_PYBUF_FLAG(FLAG) \ buf_flag = PyUnicode_FromString(#FLAG); \ + if (buf_flag == NULL) { \ + Py_DECREF(tmp); \ + return NULL; \ + } \ flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ); \ Py_DECREF(buf_flag); \ if (flag_matches == 1) { \ diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_abstract.py @@ -0,0 +1,130 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +import pytest + +class AppTestBufferProtocol(AppTestCpythonExtensionBase): + """Tests for the old buffer protocol.""" + + def w_get_buffer_support(self): + return self.import_extension('buffer_support', [ + ("charbuffer_as_string", "METH_O", + """ + char *ptr; + Py_ssize_t size; + if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize(ptr, size); + """), + ("check_readbuffer", "METH_O", + """ + return PyBool_FromLong(PyObject_CheckReadBuffer(args)); + """), + ("readbuffer_as_string", "METH_O", + """ + const void *ptr; + Py_ssize_t size; + if (PyObject_AsReadBuffer(args, &ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize((char*)ptr, size); + """), + ("writebuffer_as_string", "METH_O", + """ + void *ptr; + Py_ssize_t size; + if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize((char*)ptr, size); + """), + ("zero_out_writebuffer", "METH_O", + """ + void *ptr; + Py_ssize_t size; + Py_ssize_t i; + if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0) + return NULL; + for (i = 0; i < size; i++) { + ((char*)ptr)[i] = 0; + } + Py_RETURN_NONE; + """), + ]) + + def test_string(self): + buffer_support = self.get_buffer_support() + + s = 'a\0x' + + assert buffer_support.check_readbuffer(s) + assert s == buffer_support.readbuffer_as_string(s) + assert raises(TypeError, buffer_support.writebuffer_as_string, s) + assert s == buffer_support.charbuffer_as_string(s) + + def test_buffer(self): + buffer_support = self.get_buffer_support() + + s = 'a\0x' + buf = buffer(s) + + assert buffer_support.check_readbuffer(buf) + assert s == buffer_support.readbuffer_as_string(buf) + assert raises(TypeError, buffer_support.writebuffer_as_string, buf) + assert s == buffer_support.charbuffer_as_string(buf) + + def test_mmap(self): + import mmap + buffer_support = self.get_buffer_support() + + s = 'a\0x' + mm = mmap.mmap(-1, 3) + mm[:] = s + + assert buffer_support.check_readbuffer(mm) + assert s == buffer_support.readbuffer_as_string(mm) + assert s == buffer_support.writebuffer_as_string(mm) + assert s == buffer_support.charbuffer_as_string(mm) + + s = '\0' * 3 + buffer_support.zero_out_writebuffer(mm) + assert s == ''.join(mm) + assert s == buffer_support.readbuffer_as_string(mm) + assert s == buffer_support.writebuffer_as_string(mm) + assert s == buffer_support.charbuffer_as_string(mm) + + s = '\0' * 3 + ro_mm = mmap.mmap(-1, 3, access=mmap.ACCESS_READ) + assert buffer_support.check_readbuffer(ro_mm) + assert s == buffer_support.readbuffer_as_string(ro_mm) + assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm) + assert s == buffer_support.charbuffer_as_string(ro_mm) + + def test_array(self): + import array + buffer_support = self.get_buffer_support() + + s = 'a\0x' + a = array.array('B', [5, 0, 10]) + + buffer_support.zero_out_writebuffer(a) + assert list(a) == [0, 0, 0] + + def test_nonbuffer(self): + # e.g. int + buffer_support = self.get_buffer_support() + + assert not buffer_support.check_readbuffer(42) + assert raises(TypeError, buffer_support.readbuffer_as_string, 42) + assert raises(TypeError, buffer_support.writebuffer_as_string, 42) + assert raises(TypeError, buffer_support.charbuffer_as_string, 42) + + def test_user_class(self): + class MyBuf(str): + pass + s = 'a\0x' + buf = MyBuf(s) + buffer_support = self.get_buffer_support() + + assert buffer_support.check_readbuffer(buf) + assert s == buffer_support.readbuffer_as_string(buf) + assert raises(TypeError, buffer_support.writebuffer_as_string, buf) + assert s == buffer_support.charbuffer_as_string(buf) + + diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -0,0 +1,123 @@ +from rpython.rtyper.lltypesystem import lltype +from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.api import PyObject + +class AppTestBufferObject(AppTestCpythonExtensionBase): + + def test_FromMemory(self): + module = self.import_extension('foo', [ + ("get_FromMemory", "METH_NOARGS", + """ + cbuf = malloc(4); + cbuf[0] = 'a'; + cbuf[1] = 'b'; + cbuf[2] = 'c'; + cbuf[3] = '\\0'; + return PyBuffer_FromMemory(cbuf, 4); + """), + ("free_buffer", "METH_NOARGS", + """ + free(cbuf); + Py_RETURN_NONE; + """), + ("check_ascharbuffer", "METH_O", + """ + char *ptr; + Py_ssize_t size; + if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize(ptr, size); + """) + ], prologue = """ + static char* cbuf = NULL; + """) + buf = module.get_FromMemory() + assert str(buf) == 'abc\0' + + assert module.check_ascharbuffer(buf) == 'abc\0' + + module.free_buffer() + + def test_Buffer_New(self): + module = self.import_extension('foo', [ + ("buffer_new", "METH_NOARGS", + """ + return PyBuffer_New(150); + """), + ]) + b = module.buffer_new() + raises(AttributeError, getattr, b, 'x') + + def test_array_buffer(self): + if self.runappdirect: + skip('PyBufferObject not available outside buffer object.c') + module = self.import_extension('foo', [ + ("roundtrip", "METH_O", + """ + PyBufferObject *buf = (PyBufferObject *)args; + return PyString_FromStringAndSize(buf->b_ptr, buf->b_size); + """), + ]) + import array + a = array.array('c', 'text') + b = buffer(a) + assert module.roundtrip(b) == 'text' + + + def test_issue2752(self): + iterations = 10 + if self.runappdirect: + iterations = 2000 + module = self.import_extension('foo', [ + ("test_mod", 'METH_VARARGS', + """ + PyObject *obj; + Py_buffer bp; + if (!PyArg_ParseTuple(args, "O", &obj)) + return NULL; + + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + + if (((unsigned char*)bp.buf)[0] != '0') { + void * buf = (void*)bp.buf; + unsigned char val[4]; + char * s = PyString_AsString(obj); + memcpy(val, bp.buf, 4); + PyBuffer_Release(&bp); + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + PyErr_Format(PyExc_ValueError, + "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", + buf, val[0], val[1], val[2], val[3], + (void *)bp.buf, + ((unsigned char*)bp.buf)[0], + ((unsigned char*)bp.buf)[1], + ((unsigned char*)bp.buf)[2], + ((unsigned char*)bp.buf)[3], + s); + PyBuffer_Release(&bp); + return NULL; + } + + PyBuffer_Release(&bp); + Py_RETURN_NONE; + """), + ]) + bufsize = 4096 + def getdata(bufsize): + data = b'01234567' + for x in range(18): + data += data + if len(data) >= bufsize: + break + return data + for j in range(iterations): + block = getdata(bufsize) + assert block[:8] == '01234567' + try: + module.test_mod(block) + except ValueError as e: + print("%s at it=%d" % (e, j)) + assert False diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_intobject.py @@ -0,0 +1,247 @@ +from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase +from pypy.module.cpyext.intobject import ( + PyInt_Check, PyInt_AsLong, PyInt_AS_LONG, + PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask, + PyInt_AsUnsignedLongLongMask) +from pypy.module.cpyext.pyobject import (decref, make_ref, + get_w_obj_and_decref) +from pypy.module.cpyext.state import State +import sys + +class TestIntObject(BaseApiTest): + def test_intobject(self, space): + state = space.fromcache(State) + assert PyInt_Check(space, space.wrap(3)) + assert PyInt_Check(space, space.w_True) + assert not PyInt_Check(space, space.wrap((1, 2, 3))) + for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]: + x = PyInt_AsLong(space, space.wrap(i)) + y = PyInt_AS_LONG(space, space.wrap(i)) + assert x == i + assert y == i + py_x = state.C.PyInt_FromLong(x + 1) + w_x = get_w_obj_and_decref(space, py_x) + assert space.type(w_x) is space.w_int + assert space.eq_w(w_x, space.wrap(i + 1)) + + with raises_w(space, TypeError): + PyInt_AsLong(space, space.w_None) + + with raises_w(space, TypeError): + PyInt_AsLong(space, None) + + assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint + with raises_w(space, ValueError): + PyInt_AsUnsignedLong(space, space.wrap(-5)) + + assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint)) + == sys.maxint) + assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30)) + == 10 ** 30 % ((sys.maxint + 1) * 2)) + + assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint)) + == sys.maxint) + assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30)) + == 10 ** 30 % (2 ** 64)) + + def test_freelist_direct(self, space): + state = space.fromcache(State) + p_x = state.C.PyInt_FromLong(12345678) + decref(space, p_x) + p_y = state.C.PyInt_FromLong(87654321) + # check that the address is the same, i.e. that the freelist did its + # job + assert p_x == p_y + decref(space, p_y) + + def test_freelist_make_ref(self, space): + w_x = space.newint(12345678) + w_y = space.newint(87654321) + p_x = make_ref(space, w_x) + decref(space, p_x) + p_y = make_ref(space, w_y) + # check that the address is the same: note that w_x does NOT keep p_x + # alive, because in make_ref we have a special case for ints + assert p_x == p_y + decref(space, p_y) + + def test_freelist_int_subclass(self, space): + w_MyInt = space.appexec([], """(): + class MyInt(int): + pass + return MyInt""") + w_x = space.call_function(w_MyInt, space.newint(12345678)) + w_y = space.call_function(w_MyInt, space.newint(87654321)) + p_x = make_ref(space, w_x) + decref(space, p_x) + p_y = make_ref(space, w_y) + # now the address is different because the freelist does not work for + # int subclasses + assert p_x != p_y + decref(space, p_y) + + def test_coerce(self, space): + w_obj = space.appexec([], """(): + class Coerce(object): + def __int__(self): + return 42 + return Coerce()""") + assert PyInt_AsLong(space, w_obj) == 42 + +class AppTestIntObject(AppTestCpythonExtensionBase): + def test_fromstring(self): + module = self.import_extension('foo', [ + ("from_string", "METH_NOARGS", + """ + return PyInt_FromString("1234", NULL, 16); + """), + ]) + assert module.from_string() == 0x1234 + assert type(module.from_string()) is int + + def test_size_t(self): + module = self.import_extension('foo', [ + ("values", "METH_NOARGS", + """ + return Py_BuildValue("NNNN", + PyInt_FromSize_t(123), + PyInt_FromSize_t((size_t)-1), + PyInt_FromSsize_t(123), + PyInt_FromSsize_t((size_t)-1)); + """), + ]) + values = module.values() + types = [type(x) for x in values] + assert types == [int, long, int, int] + + def test_int_subtype(self): + module = self.import_extension( + 'foo', [ + ("newEnum", "METH_VARARGS", + """ + EnumObject *enumObj; + int intval; + PyObject *name; + + if (!PyArg_ParseTuple(args, "Oi", &name, &intval)) + return NULL; + + enumObj = PyObject_New(EnumObject, &Enum_Type); + if (!enumObj) { + return NULL; + } + + enumObj->ob_ival = intval; + Py_INCREF(name); + enumObj->ob_name = name; + + return (PyObject *)enumObj; + """), + ], + prologue=""" + #include "structmember.h" + typedef struct + { + PyObject_HEAD + long ob_ival; + PyObject* ob_name; + } EnumObject; + + static void + enum_dealloc(PyObject *op) + { + Py_DECREF(((EnumObject *)op)->ob_name); + Py_TYPE(op)->tp_free(op); + } + + static PyMemberDef enum_members[] = { + {"name", T_OBJECT, offsetof(EnumObject, ob_name), 0, NULL}, + {NULL} /* Sentinel */ + }; + + PyTypeObject Enum_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + /*tp_name*/ "Enum", + /*tp_basicsize*/ sizeof(EnumObject), + /*tp_itemsize*/ 0, + /*tp_dealloc*/ enum_dealloc, + /*tp_print*/ 0, + /*tp_getattr*/ 0, + /*tp_setattr*/ 0, + /*tp_compare*/ 0, + /*tp_repr*/ 0, + /*tp_as_number*/ 0, + /*tp_as_sequence*/ 0, + /*tp_as_mapping*/ 0, + /*tp_hash*/ 0, + /*tp_call*/ 0, + /*tp_str*/ 0, + /*tp_getattro*/ 0, + /*tp_setattro*/ 0, + /*tp_as_buffer*/ 0, + /*tp_flags*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + /*tp_doc*/ 0, + /*tp_traverse*/ 0, + /*tp_clear*/ 0, + /*tp_richcompare*/ 0, + /*tp_weaklistoffset*/ 0, + /*tp_iter*/ 0, + /*tp_iternext*/ 0, + /*tp_methods*/ 0, + /*tp_members*/ enum_members, + /*tp_getset*/ 0, + /*tp_base*/ 0, /* set to &PyInt_Type in init function for MSVC */ + /*tp_dict*/ 0, + /*tp_descr_get*/ 0, + /*tp_descr_set*/ 0, + /*tp_dictoffset*/ 0, + /*tp_init*/ 0, + /*tp_alloc*/ 0, + /*tp_new*/ 0 + }; + """, more_init = ''' + Enum_Type.tp_base = &PyInt_Type; + if (PyType_Ready(&Enum_Type) < 0) INITERROR; + ''') + + a = module.newEnum("ULTIMATE_ANSWER", 42) + assert type(a).__name__ == "Enum" + assert isinstance(a, int) + assert a == int(a) == 42 + assert a.name == "ULTIMATE_ANSWER" + + def test_int_cast(self): + mod = self.import_extension('foo', [ + #prove it works for ints + ("test_int", "METH_NOARGS", + """ + PyObject * obj = PyInt_FromLong(42); + PyObject * val; + if (!PyInt_Check(obj)) { + Py_DECREF(obj); + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival); + Py_DECREF(obj); + return val; + """ + ), + ]) + i = mod.test_int() + assert isinstance(i, int) + assert i == 42 + + def test_int_macros(self): + mod = self.import_extension('foo', [ + ("test_macros", "METH_NOARGS", + """ + PyObject * obj = PyInt_FromLong(42); + PyIntObject * i = (PyIntObject*)obj; + PyInt_AS_LONG(obj); + PyInt_AS_LONG(i); + Py_RETURN_NONE; + """ + ), + ]) diff --git a/pypy/module/cpyext/test/test_pycobject.py b/pypy/module/cpyext/test/test_pycobject.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_pycobject.py @@ -0,0 +1,30 @@ +import py +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + +class AppTestStringObject(AppTestCpythonExtensionBase): + def test_pycobject_import(self): + module = self.import_extension('foo', [ + ("set_ptr", "METH_O", + """ + PyObject *pointer, *module; + void *ptr = PyLong_AsVoidPtr(args); + if (PyErr_Occurred()) return NULL; + pointer = PyCObject_FromVoidPtr(ptr, NULL); + if (PyErr_Occurred()) return NULL; + module = PyImport_ImportModule("foo"); + PyModule_AddObject(module, "_ptr", pointer); + Py_DECREF(module); + if (PyErr_Occurred()) return NULL; + Py_RETURN_NONE; + """), + ("get_ptr", "METH_NOARGS", + """ + void *ptr = PyCObject_Import("foo", "_ptr"); + if (PyErr_Occurred()) return NULL; + return PyLong_FromVoidPtr(ptr); + """)]) + module.set_ptr(1234) + assert "PyCObject object" in str(module._ptr) + import gc; gc.collect() + assert module.get_ptr() == 1234 + del module._ptr diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -428,6 +428,9 @@ def utf8_w(self, space): return self._value + def utf8_w(self, space): + return self._value + def buffer_w(self, space, flags): space.check_buf_flags(flags, True) return SimpleView(StringBuffer(self._value)) diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -460,7 +460,7 @@ ptr = self.start_ptr if not self.next_char_ok(ctx, pattern, ptr, self.ppos3): return - self.start_ptr = ptr + 1 + self.start_ptr = ctx.next(ptr) return self.find_first_result(ctx, pattern) def next_char_ok(self, ctx, pattern, ptr, ppos): @@ -736,9 +736,9 @@ startptr, length_bytes = get_group_ref(ctx, marks, pattern.pat(ppos)) if length_bytes < 0: return # group was not previously defined - if not match_repeated_ignore(ctx, ptr, startptr, length_bytes): + ptr = match_repeated_ignore(ctx, ptr, startptr, length_bytes) + if ptr < ctx.ZERO: return # no match - ptr = ctx.go_forward_by_bytes(ptr, length_bytes) ppos += 1 elif op == OPCODE_GROUPREF_EXISTS: diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -66,12 +66,12 @@ return position_high - position_low -def make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags): +def make_utf8_ctx(utf8string, bytestart, byteend, flags): if bytestart < 0: bytestart = 0 elif bytestart > len(utf8string): bytestart = len(utf8string) if byteend < 0: byteend = 0 elif byteend > len(utf8string): byteend = len(utf8string) - ctx = Utf8MatchContext(pattern, utf8string, bytestart, byteend, flags) + ctx = Utf8MatchContext(utf8string, bytestart, byteend, flags) ctx.debug_check_pos(bytestart) ctx.debug_check_pos(byteend) return ctx @@ -81,8 +81,8 @@ # utf8string. from rpython.rlib.rsre.rsre_core import search_context - ctx = make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags) - if search_context(ctx): + ctx = make_utf8_ctx(utf8string, bytestart, byteend, flags) + if search_context(ctx, pattern): return ctx else: return None @@ -93,9 +93,9 @@ # utf8string. from rpython.rlib.rsre.rsre_core import match_context - ctx = make_utf8_ctx(pattern, utf8string, bytestart, byteend, flags) + ctx = make_utf8_ctx(utf8string, bytestart, byteend, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None From pypy.commits at gmail.com Sun Nov 4 16:52:05 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Nov 2018 13:52:05 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5bdf6a05.1c69fb81.df574.2130@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95273:249385a41d6e Date: 2018-11-04 11:05 -0500 http://bitbucket.org/pypy/pypy/changeset/249385a41d6e/ Log: merge default into branch diff too long, truncating to 2000 out of 2058 lines diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -396,17 +396,21 @@ return archive_name -def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False): +def _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger): # XXX see if we want to keep an external call here if verbose: zipoptions = "-r" else: zipoptions = "-rq" - from distutils.errors import DistutilsExecError - from distutils.spawn import spawn + cmd = ["zip", zipoptions, zip_filename, base_dir] + if logger is not None: + logger.info(' '.join(cmd)) + if dry_run: + return + import subprocess try: - spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) - except DistutilsExecError: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: # XXX really should distinguish between "couldn't find # external 'zip' command" and "zip failed". raise ExecError, \ @@ -440,7 +444,7 @@ zipfile = None if zipfile is None: - _call_external_zip(base_dir, zip_filename, verbose, dry_run) + _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger) else: if logger is not None: logger.info("creating '%s' and adding '%s' to it", diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py --- a/lib-python/2.7/test/test_inspect.py +++ b/lib-python/2.7/test/test_inspect.py @@ -45,6 +45,9 @@ git = mod.StupidGit() +class ExampleClassWithSlot(object): + __slots__ = 'myslot' + class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, @@ -96,7 +99,11 @@ else: self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals)) if hasattr(types, 'MemberDescriptorType'): - self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals') + # App-level slots are member descriptors on both PyPy and + # CPython, but the various built-in attributes are all + # getsetdescriptors on PyPy. So check ismemberdescriptor() + # with an app-level slot. + self.istest(inspect.ismemberdescriptor, 'ExampleClassWithSlot.myslot') else: self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals)) diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -486,6 +486,8 @@ return cobj, cobj._to_ffi_param(), type(cobj) def _convert_args_for_callback(self, argtypes, args): + from _ctypes.structure import StructOrUnion + # assert len(argtypes) == len(args) newargs = [] for argtype, arg in zip(argtypes, args): @@ -495,6 +497,10 @@ param = param._get_buffer_value() elif self._is_primitive(argtype): param = param.value + elif isinstance(param, StructOrUnion): # not a *pointer* to struct + newparam = StructOrUnion.__new__(type(param)) + param._copy_to(newparam._buffer.buffer) + param = newparam newargs.append(param) return newargs diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -162,6 +162,17 @@ module_path = module_name.split('.') module_path[-1] += '.py' generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules dist.cmdclass['build_py'] = build_py_make_mod # distutils and setuptools have no notion I could find of a @@ -171,6 +182,7 @@ # the module. So we add it here, which gives a few apparently # harmless warnings about not finding the file outside the # build directory. + # Then we need to hack more in get_source_files(); see above. if dist.py_modules is None: dist.py_modules = [] dist.py_modules.append(module_name) diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -26,6 +26,12 @@ from pyrepl.fancy_termios import tcgetattr, tcsetattr from pyrepl.console import Console, Event from pyrepl import unix_eventqueue +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -70,8 +76,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -416,6 +422,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError), err: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -36,7 +36,22 @@ Small refactorings in the Python parser. +.. branch: fix-readme-typo + +.. branch: avoid_shell_injection_in_shutil + .. branch: unicode-utf8-re .. branch: utf8-io Utf8 handling for unicode + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -136,89 +136,89 @@ max_jitdriver = jit.JitDriver(name='max', greens=['has_key', 'has_item', 'w_type'], reds='auto') -def make_min_max(unroll): - @specialize.arg(2) - def min_max_impl(space, args, implementation_of): - if implementation_of == "max": - compare = space.gt - jitdriver = max_jitdriver + at specialize.arg(3) +def min_max_sequence(space, w_sequence, w_key, implementation_of): + if implementation_of == "max": + compare = space.gt + jitdriver = max_jitdriver + else: + compare = space.lt + jitdriver = min_jitdriver + w_iter = space.iter(w_sequence) + w_type = space.type(w_iter) + has_key = w_key is not None + has_item = False + w_max_item = None + w_max_val = None + while True: + jitdriver.jit_merge_point(has_key=has_key, has_item=has_item, + w_type=w_type) + try: + w_item = space.next(w_iter) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + break + if has_key: + w_compare_with = space.call_function(w_key, w_item) else: - compare = space.lt - jitdriver = min_jitdriver - any_kwds = bool(args.keywords) - args_w = args.arguments_w - if len(args_w) > 1: - if unroll and len(args_w) == 2 and not any_kwds: - # a fast path for the common case, useful for interpreted - # mode and to reduce the length of the jit trace - w0, w1 = args_w - if space.is_true(compare(w1, w0)): - return w1 - else: - return w0 - w_sequence = space.newtuple(args_w) - elif len(args_w): - w_sequence = args_w[0] + w_compare_with = w_item + if (not has_item or + space.is_true(compare(w_compare_with, w_max_val))): + has_item = True + w_max_item = w_item + w_max_val = w_compare_with + if w_max_item is None: + raise oefmt(space.w_ValueError, "arg is an empty sequence") + return w_max_item + + at specialize.arg(3) + at jit.look_inside_iff(lambda space, args_w, w_key, implementation_of: + jit.loop_unrolling_heuristic(args_w, len(args_w), 3)) +def min_max_multiple_args(space, args_w, w_key, implementation_of): + # case of multiple arguments (at least two). We unroll it if there + # are 2 or 3 arguments. + if implementation_of == "max": + compare = space.gt + else: + compare = space.lt + w_max_item = args_w[0] + if w_key is not None: + w_max_val = space.call_function(w_key, w_max_item) + else: + w_max_val = w_max_item + for i in range(1, len(args_w)): + w_item = args_w[i] + if w_key is not None: + w_compare_with = space.call_function(w_key, w_item) else: - raise oefmt(space.w_TypeError, - "%s() expects at least one argument", - implementation_of) - w_key = None - if any_kwds: - kwds = args.keywords - if kwds[0] == "key" and len(kwds) == 1: - w_key = args.keywords_w[0] - else: - raise oefmt(space.w_TypeError, - "%s() got unexpected keyword argument", - implementation_of) - - w_iter = space.iter(w_sequence) - w_type = space.type(w_iter) - has_key = w_key is not None - has_item = False - w_max_item = None - w_max_val = None - while True: - if not unroll: - jitdriver.jit_merge_point(has_key=has_key, has_item=has_item, w_type=w_type) - try: - w_item = space.next(w_iter) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - break - if has_key: - w_compare_with = space.call_function(w_key, w_item) - else: - w_compare_with = w_item - if not has_item or \ - space.is_true(compare(w_compare_with, w_max_val)): - has_item = True - w_max_item = w_item - w_max_val = w_compare_with - if w_max_item is None: - raise oefmt(space.w_ValueError, "arg is an empty sequence") - return w_max_item - if unroll: - min_max_impl = jit.unroll_safe(min_max_impl) - return min_max_impl - -min_max_unroll = make_min_max(True) -min_max_normal = make_min_max(False) + w_compare_with = w_item + if space.is_true(compare(w_compare_with, w_max_val)): + w_max_item = w_item + w_max_val = w_compare_with + return w_max_item @specialize.arg(2) def min_max(space, args, implementation_of): - # the 'normal' version includes a JIT merge point, which will make a - # new loop (from the interpreter or from another JIT loop). If we - # give exactly two arguments to the call to max(), or a JIT virtual - # list of arguments, then we pick the 'unroll' version with no JIT - # merge point. - if jit.isvirtual(args.arguments_w) or len(args.arguments_w) == 2: - return min_max_unroll(space, args, implementation_of) + w_key = None + if bool(args.keywords): + kwds = args.keywords + if kwds[0] == "key" and len(kwds) == 1: + w_key = args.keywords_w[0] + else: + raise oefmt(space.w_TypeError, + "%s() got unexpected keyword argument", + implementation_of) + # + args_w = args.arguments_w + if len(args_w) > 1: + return min_max_multiple_args(space, args_w, w_key, implementation_of) + elif len(args_w): + return min_max_sequence(space, args_w[0], w_key, implementation_of) else: - return min_max_normal(space, args, implementation_of) -min_max._always_inline = True + raise oefmt(space.w_TypeError, + "%s() expects at least one argument", + implementation_of) def max(space, __args__): """max(iterable[, key=func]) -> value diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -353,6 +353,10 @@ assert type(max(1.0, 1L, 1)) is float assert type(max(1L, 1, 1.0)) is long + def test_max_list_and_key(self): + assert max(["100", "50", "30", "-200"], key=int) == "100" + assert max("100", "50", "30", "-200", key=int) == "100" + try: from hypothesis import given, strategies, example diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -109,6 +109,7 @@ '_promote' : 'interp_magic._promote', 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -211,3 +211,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,23 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -383,3 +392,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3957,6 +3957,7 @@ z3 = cast(BVoidP, 0) z4 = cast(BUCharP, 0) with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") newp(new_pointer_type(BIntP), z1) # warn assert len(w) == 1 newp(new_pointer_type(BVoidP), z1) # fine diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,15 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -603,13 +603,13 @@ def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) - return (u"\x01", 4) + return (u"\x01", 5) codecs.register_error("test.hui", handler_unicodeinternal) res = "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui") if sys.maxunicode > 65535: assert res == u"\u0000\u0001" # UCS4 build else: - assert res == u"\x00\x00\x01\x00\x00" # UCS2 build + assert res == u"\x00\x00\x01" # UCS2 build def handler1(exc): if not isinstance(exc, UnicodeEncodeError) \ @@ -621,6 +621,26 @@ assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: + "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -92,8 +92,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -603,10 +606,11 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyObject_GetBuffer', 'PyBuffer_Release', - 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', - 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', '_Py_get_buffer_type', - '_Py_setfilesystemdefaultencoding', + 'PyObject_DelItemString', 'PyObject_GetBuffer', 'PyBuffer_Release', + + 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', + 'PyBuffer_FromObject', 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', + 'PyBuffer_Type', '_Py_get_buffer_type', '_Py_setfilesystemdefaultencoding', 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', @@ -640,6 +644,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', @@ -1176,6 +1181,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1299,7 +1308,7 @@ # if do tuple_attach of the prebuilt empty tuple, we need to call # _PyPy_Malloc) builder.attach_all(space) - + setup_init_functions(eci, prefix) return modulename.new(ext='') @@ -1536,7 +1545,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1550,6 +1558,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = ['#include ', + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, @@ -1655,7 +1666,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: @@ -1716,6 +1731,12 @@ w_mod = state.fixup_extension(name, path) return w_mod +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -4,6 +4,15 @@ extern "C" { #endif + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); + + /* + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ + + /* new buffer API */ #define PyObject_CheckBuffer(obj) \ @@ -28,6 +37,27 @@ /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ +/* Mapping protocol:*/ + + /* implemented as a macro: + + int PyMapping_DelItemString(PyObject *o, char *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItemString(O,K) PyObject_DelItemString((O),(K)) + + /* implemented as a macro: + + int PyMapping_DelItem(PyObject *o, PyObject *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -47,6 +47,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -23,6 +23,23 @@ /* Operations on any object */ int +PyObject_DelItemString(PyObject *o, char *key) +{ + PyObject *okey; + int ret; + + if (o == NULL || key == NULL) { + null_error(); + return -1; + } + okey = PyString_FromString(key); + if (okey == NULL) + return -1; + ret = PyObject_DelItem(o, okey); + Py_DECREF(okey); + return ret; +} +int PyObject_CheckReadBuffer(PyObject *obj) { PyBufferProcs *pb = obj->ob_type->tp_as_buffer; @@ -101,6 +118,20 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) { @@ -116,6 +147,7 @@ return (void*)pointer; } + void _Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape) { @@ -258,19 +290,6 @@ -/* Buffer C-API for Python 3.0 */ - -int -PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) -{ - if (!PyObject_CheckBuffer(obj)) { - PyErr_Format(PyExc_TypeError, - "'%100s' does not have the buffer interface", - Py_TYPE(obj)->tp_name); - return -1; - } - return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); -} void PyBuffer_Release(Py_buffer *view) @@ -428,6 +447,7 @@ return retval; } + static PyObject * objargs_mktuple(va_list va) { diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,28 +1,11 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFileObject = rffi.VOIDP -PyCodeObject = rffi.VOIDP -PyFrameObject = rffi.VOIDP -_inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP -_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype + @cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL) def PyBuffer_SizeFromFormat(space, format): @@ -254,39 +237,6 @@ instead.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP], rffi.DOUBLE, error=CANNOT_FAIL) def PyOS_ascii_atof(space, nptr): """Convert a string to a double in a locale-independent way. @@ -310,24 +260,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -1206,14 +1138,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.ULONGLONG, error=-1) -def PyInt_AsUnsignedLongLongMask(space, io): - """Will first attempt to cast the object to a PyIntObject or - PyLongObject, if it is not already one, and then return its value as - unsigned long long, without checking for overflow. - """ - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyInt_ClearFreeList(space): """Clear the integer free list. Return the number of items that could not @@ -1233,18 +1157,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) -def PyMapping_DelItemString(space, o, key): - """Remove the mapping for object key from the object o. Return -1 on - failure. This is equivalent to the Python statement del o[key].""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyMapping_DelItem(space, o, key): - """Remove the mapping for object key from the object o. Return -1 on - failure. This is equivalent to the Python statement del o[key].""" - raise NotImplementedError - @cpython_api([lltype.Signed, FILE, rffi.INT_real], lltype.Void) def PyMarshal_WriteLongToFile(space, value, file, version): """Marshal a long integer, value, to file. This will only write @@ -1345,13 +1257,6 @@ for PyObject_Str().""" raise NotImplementedError - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - borrow_from() - raise NotImplementedError - @cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -420,3 +420,15 @@ except StopIteration: pass assert out == [0, 1, 2, 3, 4] + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyInt_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py --- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py +++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py @@ -30,37 +30,42 @@ sa = 0 while i < 30000: lst = range(i % 1000 + 2) - sa += max(*lst) # ID: max + sa += max(*lst) # ID: callmax i += 1 return sa log = self.run(main, []) assert log.result == main() loop, = log.loops_by_filename(self.filepath) - assert loop.match(""" + assert loop.match_by_id('callmax', """ ... - p76 = call_assembler_r(_, _, _, _, descr=...) + p76 = call_may_force_r(_, _, _, _, descr=...) ... """) - loop2 = log.loops[0] - loop2.match(''' - ... - label(..., descr=...) - ... - label(..., descr=...) - guard_not_invalidated? - i17 = int_ge(i11, i7) - guard_false(i17, descr=...) - p18 = getarrayitem_gc_r(p5, i11, descr=...) - i19 = int_add(i11, 1) - setfield_gc(p2, i19, descr=...) - guard_nonnull_class(p18, ConstClass(W_IntObject), descr=...) - i20 = getfield_gc_i(p18, descr=...) - i21 = int_gt(i20, i14) - guard_true(i21, descr=...) - jump(..., descr=...) - ''') - # XXX could be "guard_class(p18)" instead; we lost somewhere - # the information that it cannot be null. + + #----- the following logic used to check the content of the assembly + #----- generated for the loop in max(), but now we no longer produce + #----- any custom assembly in this case. It used to say + #----- 'call_assembler_r' above, and now it says 'call_may_force_r'. + #loop2 = log.loops[0] + #loop2.match(''' + #... + #label(..., descr=...) + #... + #label(..., descr=...) + #guard_not_invalidated? + #i17 = int_ge(i11, i7) + #guard_false(i17, descr=...) + #p18 = getarrayitem_gc_r(p5, i11, descr=...) + #i19 = int_add(i11, 1) + #setfield_gc(p2, i19, descr=...) + #guard_nonnull_class(p18, ConstClass(W_IntObject), descr=...) + #i20 = getfield_gc_i(p18, descr=...) + #i21 = int_gt(i20, i14) + #guard_true(i21, descr=...) + #jump(..., descr=...) + #''') + ## XXX could be "guard_class(p18)" instead; we lost somewhere + ## the information that it cannot be null. def test_iter_max(self): def main(): diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -183,7 +183,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1387,6 +1387,7 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("enum foo;") with warnings.catch_warnings(record=True) as log: + warnings.simplefilter("always") n = ffi.cast("enum foo", -1) assert int(n) == 0xffffffff assert str(log[0].message) == ( diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -46,14 +46,14 @@ assert x != math.sin(1.23) # rounding effects assert abs(x - math.sin(1.23)) < 1E-6 - def test_sin_no_return_value(self): + def test_lround_no_return_value(self): # check that 'void'-returning functions work too ffi = FFI(backend=self.Backend()) ffi.cdef(""" - void sin(double x); + void lround(double x); """) m = ffi.dlopen(lib_m) - x = m.sin(1.23) + x = m.lround(1.23) assert x is None def test_dlopen_filename(self): diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py @@ -160,15 +160,17 @@ proto = CFUNCTYPE(c_int, RECT) def callback(point): + point.left *= -1 return point.left+point.top+point.right+point.bottom cbp = proto(callback) - rect = RECT(1000,100,10,1) + rect = RECT(-1000,100,10,1) res = cbp(rect) assert res == 1111 + assert rect.left == -1000 # must not have been changed! def test_callback_from_c_with_struct_argument(self): import conftest diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py --- a/pypy/module/thread/__init__.py +++ b/pypy/module/thread/__init__.py @@ -18,6 +18,7 @@ 'allocate_lock': 'os_lock.allocate_lock', 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.Lock', + 'RLock': 'os_lock.W_RLock', # pypy only, issue #2905 '_local': 'os_local.Local', 'error': 'space.fromcache(error.Cache).w_error', } diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -8,8 +8,8 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, make_weakref_descr -from pypy.interpreter.error import oefmt -from rpython.rlib.rarithmetic import r_longlong, ovfcheck_float_to_longlong +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rarithmetic import r_longlong, ovfcheck, ovfcheck_float_to_longlong RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3) @@ -53,6 +53,12 @@ break return result +def try_release(space, lock): + try: + lock.release() + except rthread.error: + raise wrap_thread_error(space, "release unlocked lock") + class Lock(W_Root): "A box around an interp-level lock object." @@ -97,10 +103,7 @@ """Release the lock, allowing another thread that is blocked waiting for the lock to acquire the lock. The lock must be in the locked state, but it needn't be locked by the same thread that unlocks it.""" - try: - self.lock.release() - except rthread.error: - raise wrap_thread_error(space, "release unlocked lock") + try_release(space, self.lock) def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" @@ -162,3 +165,149 @@ """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" return Lock(space) + +class W_RLock(W_Root): + # Does not exist in CPython 2.x. Back-ported from PyPy3. See issue #2905 + + def __init__(self, space, w_active=None): + self.rlock_count = 0 + self.rlock_owner = 0 + self.w_active = w_active # dictionary 'threading._active' + try: + self.lock = rthread.allocate_lock() + except rthread.error: + raise wrap_thread_error(space, "cannot allocate lock") + + def descr__new__(space, w_subtype, w_active=None): + self = space.allocate_instance(W_RLock, w_subtype) + W_RLock.__init__(self, space, w_active) + return self + + def descr__repr__(self, space): + w_type = space.type(self) + classname = w_type.name + if self.rlock_owner == 0: + owner = "None" + else: + owner = str(self.rlock_owner) + if self.w_active is not None: + try: + w_owner = space.getitem(self.w_active, + space.newint(self.rlock_owner)) + w_name = space.getattr(w_owner, space.newtext('name')) + owner = space.text_w(space.repr(w_name)) + except OperationError as e: + if e.async(space): + raise + return space.newtext("<%s owner=%s count=%d>" % ( + classname, owner, self.rlock_count)) + + @unwrap_spec(blocking=int) + def acquire_w(self, space, blocking=1): + """Acquire a lock, blocking or non-blocking. + + When invoked without arguments: if this thread already owns the lock, + increment the recursion level by one, and return immediately. Otherwise, + if another thread owns the lock, block until the lock is unlocked. Once + the lock is unlocked (not owned by any thread), then grab ownership, set + the recursion level to one, and return. If more than one thread is + blocked waiting until the lock is unlocked, only one at a time will be + able to grab ownership of the lock. There is no return value in this + case. + + When invoked with the blocking argument set to true, do the same thing + as when called without arguments, and return true. + + When invoked with the blocking argument set to false, do not block. If a + call without an argument would block, return false immediately; + otherwise, do the same thing as when called without arguments, and + return true. + + """ + tid = rthread.get_ident() + if tid == self.rlock_owner: + try: + self.rlock_count = ovfcheck(self.rlock_count + 1) + except OverflowError: + raise oefmt(space.w_OverflowError, + "internal lock count overflowed") + return space.w_True + + rc = self.lock.acquire(blocking != 0) + if rc: + self.rlock_owner = tid + self.rlock_count = 1 + return space.newbool(rc) + + def release_w(self, space): + """Release a lock, decrementing the recursion level. + + If after the decrement it is zero, reset the lock to unlocked (not owned + by any thread), and if any other threads are blocked waiting for the + lock to become unlocked, allow exactly one of them to proceed. If after + the decrement the recursion level is still nonzero, the lock remains + locked and owned by the calling thread. + + Only call this method when the calling thread owns the lock. A + RuntimeError is raised if this method is called when the lock is + unlocked. + + There is no return value. + + """ + if self.rlock_owner != rthread.get_ident(): + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + self.rlock_count -= 1 + if self.rlock_count == 0: + self.rlock_owner = 0 + try_release(space, self.lock) + + def is_owned_w(self, space): + """For internal use by `threading.Condition`.""" + return space.newbool(self.rlock_owner == rthread.get_ident()) + + def acquire_restore_w(self, space, w_count_owner): + """For internal use by `threading.Condition`.""" + # saved_state is the value returned by release_save() + w_count, w_owner = space.unpackiterable(w_count_owner, 2) + count = space.int_w(w_count) + owner = space.int_w(w_owner) + self.lock.acquire(True) + self.rlock_count = count + self.rlock_owner = owner + + def release_save_w(self, space): + """For internal use by `threading.Condition`.""" + if self.rlock_count == 0: + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + count, self.rlock_count = self.rlock_count, 0 + owner, self.rlock_owner = self.rlock_owner, 0 + try_release(space, self.lock) + return space.newtuple([space.newint(count), space.newint(owner)]) + + def descr__enter__(self, space): + self.acquire_w(space) + return self + + def descr__exit__(self, space, __args__): + self.release_w(space) + + def descr__note(self, space, __args__): + pass # compatibility with the _Verbose base class in Python + +W_RLock.typedef = TypeDef( + "thread.RLock", + __new__ = interp2app(W_RLock.descr__new__.im_func), + acquire = interp2app(W_RLock.acquire_w), + release = interp2app(W_RLock.release_w), + _is_owned = interp2app(W_RLock.is_owned_w), + _acquire_restore = interp2app(W_RLock.acquire_restore_w), + _release_save = interp2app(W_RLock.release_save_w), + __enter__ = interp2app(W_RLock.descr__enter__), + __exit__ = interp2app(W_RLock.descr__exit__), + __weakref__ = make_weakref_descr(W_RLock), + __repr__ = interp2app(W_RLock.descr__repr__), + _note = interp2app(W_RLock.descr__note), + ) diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -138,6 +138,62 @@ test_lock_again = AppTestLock.test_lock.im_func +class AppTestRLock(GenericTestThread): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + import thread + lock = thread.RLock() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + import thread + lock = thread.RLock() + raises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + raises(RuntimeError, lock.release) + + def test_release_save(self): + import thread + lock = thread.RLock() + raises(RuntimeError, lock._release_save) + lock.acquire() + state = lock._release_save() + lock._acquire_restore(state) + lock.release() + + def test__is_owned(self): + import thread + lock = thread.RLock() + assert lock._is_owned() is False + lock.acquire() + assert lock._is_owned() is True + lock.acquire() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is False + + def test_context_manager(self): + import thread + lock = thread.RLock() + with lock: + assert lock._is_owned() is True + + class AppTestLockSignals(GenericTestThread): pytestmark = py.test.mark.skipif("os.name != 'posix'") @@ -178,6 +234,10 @@ import thread self.acquire_retries_on_intr(thread.allocate_lock()) + def test_rlock_acquire_retries_on_intr(self): + import thread + self.acquire_retries_on_intr(thread.RLock()) + def w_alarm_interrupt(self, sig, frame): raise KeyboardInterrupt @@ -209,3 +269,20 @@ assert dt < 8.0 finally: signal.signal(signal.SIGALRM, oldalrm) + + +class AppTestLockRepr(GenericTestThread): + + def test_rlock_repr(self): + import thread + class MyThread: + name = "foobar" + actives = {thread.get_ident(): MyThread()} + rlock = thread.RLock(actives) + assert repr(rlock) == "" + rlock.acquire() + rlock.acquire() + assert repr(rlock) == "" + actives.clear() + assert repr(rlock) == "" % ( + thread.get_ident(),) diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -151,7 +151,7 @@ if getattr(func, '_elidable_function_', False): raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) if not getattr(func, '_jit_look_inside_', True): - raise TypeError("it does not make sense for %s to be both elidable and dont_look_inside" % func) + raise TypeError("it does not make sense for %s to be both unroll_safe and dont_look_inside" % func) func._jit_unroll_safe_ = True return func diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,25 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -730,16 +734,21 @@ length = rwin32.MAX_PATH + 1 traits = _preferred_traits(path) win32traits = make_win32_traits(traits) - with traits.scoped_alloc_buffer(length) as buf: - res = win32traits.GetFullPathName( - traits.as_str0(path), rffi.cast(rwin32.DWORD, length), - buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) - if res == 0: - raise rwin32.lastSavedWindowsError("_getfullpathname failed") - result = buf.str(intmask(res)) - assert result is not None - result = rstring.assert_str0(result) - return result + while True: # should run the loop body maximum twice + with traits.scoped_alloc_buffer(length) as buf: + res = win32traits.GetFullPathName( + traits.as_str0(path), rffi.cast(rwin32.DWORD, length), + buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) + res = intmask(res) + if res == 0: + raise rwin32.lastSavedWindowsError("_getfullpathname failed") + if res >= length: + length = res + 1 + continue + result = buf.str(res) + assert result is not None + result = rstring.assert_str0(result) + return result c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP, diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1777,8 +1777,6 @@ "truncated input", s, pos, size) result.append(res) - if pos > size - unicode_bytes: - break continue t = r_uint(0) h = 0 diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,12 +196,28 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, NULL_HANDLE, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -215,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] diff --git a/rpython/rlib/test/loadtest/loadtest1.dll b/rpython/rlib/test/loadtest/loadtest1.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cb83854875c876717371bdf90488eed9c6571f03 GIT binary patch [cut] diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -83,6 +83,14 @@ # the most intriguing failure of ntpath.py should not repeat, here: assert not data.endswith(stuff) + @win_only + def test__getfullpathname_long(self): + stuff = "C:" + "\\abcd" * 100 + py.test.raises(WindowsError, rposix.getfullpathname, stuff) + ustuff = u"C:" + u"\\abcd" * 100 + res = rposix.getfullpathname(ustuff) + assert res == ustuff + def test_getcwd(self): assert rposix.getcwd() == os.getcwd() diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -6,6 +6,44 @@ from rpython.rlib import rwin32 from rpython.tool.udir import udir +loadtest_dir = os.path.dirname(__file__) + '/loadtest' +test1 = os.path.abspath(loadtest_dir + '/loadtest1.dll') +test0 = os.path.abspath(loadtest_dir + '/loadtest0.dll') + +if not os.path.exists(test1) or not os.path.exists(test0): + # This is how the files, which are checked into the repo, were created + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform + from rpython.translator import cdir + if not os.path.exists(loadtest_dir): + os.mkdir(loadtest_dir) + c_file = udir.ensure("test_rwin32", dir=1).join("test0.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + RPY_EXPORTED + int internal_sum(int a, int b) { + return a + b; + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir]) + lib_name = str(platform.compile([c_file], eci, test0[:-4], + standalone=False)) + assert os.path.abspath(lib_name) == os.path.abspath(test0) + + c_file = udir.ensure("test_rwin32", dir=1).join("test1.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + int internal_sum(int a, int b); + RPY_EXPORTED + int sum(int a, int b) { + return internal_sum(a, b); + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir], + libraries=[loadtest_dir + '/loadtest0']) + lib_name = str(platform.compile([c_file], eci, test1[:-4], + standalone=False, )) + assert os.path.abspath(lib_name) == os.path.abspath(test1) def test_get_osfhandle(): fid = open(str(udir.join('validate_test.txt')), 'w') @@ -28,13 +66,13 @@ "import time;" "time.sleep(10)", ], - ) + ) print proc.pid handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, proc.pid) assert rwin32.TerminateProcess(handle, signal.SIGTERM) == 1 rwin32.CloseHandle(handle) assert proc.wait() == signal.SIGTERM - + @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive') def test_wenviron(): name, value = u'PYPY_TEST_日本', u'foobar日本' @@ -55,3 +93,48 @@ msg = rwin32.FormatErrorW(34) assert type(msg) is unicode assert u'%2' in msg + +def test_loadlibraryA(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibrary(test0) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibrary(test1) + assert not hdll + + assert os.path.exists(test1) + + hdll = rwin32.LoadLibraryExA(test1) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) From pypy.commits at gmail.com Sun Nov 4 16:52:08 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Nov 2018 13:52:08 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: more tightly qualify check Message-ID: <5bdf6a08.1c69fb81.f211c.8c20@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95274:b2e29706f57a Date: 2018-11-04 12:05 -0500 http://bitbucket.org/pypy/pypy/changeset/b2e29706f57a/ Log: more tightly qualify check diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1247,7 +1247,7 @@ value = function(*args) assert isinstance(value, int) else: - if not value: + if not isinstance(value, list) and not value: value = function(*args) assert not isinstance(value, int) return value From pypy.commits at gmail.com Sun Nov 4 17:05:24 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Nov 2018 14:05:24 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge unicode-utf8 into branch Message-ID: <5bdf6d24.1c69fb81.5b562.d9d1@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95275:07764fd85832 Date: 2018-11-04 17:04 -0500 http://bitbucket.org/pypy/pypy/changeset/07764fd85832/ Log: merge unicode-utf8 into branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -38,6 +38,8 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + .. branch: unicode-utf8-re .. branch: utf8-io @@ -45,4 +47,15 @@ Utf8 handling for unicode .. branch: pyparser-improvements-3 -Small refactorings in the Python parser. \ No newline at end of file +Small refactorings in the Python parser. + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` \ No newline at end of file diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -786,6 +786,26 @@ return (u"\x01", 4) # 4 < len(input), so will try and fail again codecs.register_error("test.inf", handler_unicodeinternal) try: + "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") except MyException: pass diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_abstract.py +++ /dev/null @@ -1,130 +0,0 @@ -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -import pytest - -class AppTestBufferProtocol(AppTestCpythonExtensionBase): - """Tests for the old buffer protocol.""" - - def w_get_buffer_support(self): - return self.import_extension('buffer_support', [ - ("charbuffer_as_string", "METH_O", - """ - char *ptr; - Py_ssize_t size; - if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0) - return NULL; - return PyString_FromStringAndSize(ptr, size); - """), - ("check_readbuffer", "METH_O", - """ - return PyBool_FromLong(PyObject_CheckReadBuffer(args)); - """), - ("readbuffer_as_string", "METH_O", - """ - const void *ptr; - Py_ssize_t size; - if (PyObject_AsReadBuffer(args, &ptr, &size) < 0) - return NULL; - return PyString_FromStringAndSize((char*)ptr, size); - """), - ("writebuffer_as_string", "METH_O", - """ - void *ptr; - Py_ssize_t size; - if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0) - return NULL; - return PyString_FromStringAndSize((char*)ptr, size); - """), - ("zero_out_writebuffer", "METH_O", - """ - void *ptr; - Py_ssize_t size; - Py_ssize_t i; - if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0) - return NULL; - for (i = 0; i < size; i++) { - ((char*)ptr)[i] = 0; - } - Py_RETURN_NONE; - """), - ]) - - def test_string(self): - buffer_support = self.get_buffer_support() - - s = 'a\0x' - - assert buffer_support.check_readbuffer(s) - assert s == buffer_support.readbuffer_as_string(s) - assert raises(TypeError, buffer_support.writebuffer_as_string, s) - assert s == buffer_support.charbuffer_as_string(s) - - def test_buffer(self): - buffer_support = self.get_buffer_support() - - s = 'a\0x' - buf = buffer(s) - - assert buffer_support.check_readbuffer(buf) - assert s == buffer_support.readbuffer_as_string(buf) - assert raises(TypeError, buffer_support.writebuffer_as_string, buf) - assert s == buffer_support.charbuffer_as_string(buf) - - def test_mmap(self): - import mmap - buffer_support = self.get_buffer_support() - - s = 'a\0x' - mm = mmap.mmap(-1, 3) - mm[:] = s - - assert buffer_support.check_readbuffer(mm) - assert s == buffer_support.readbuffer_as_string(mm) - assert s == buffer_support.writebuffer_as_string(mm) - assert s == buffer_support.charbuffer_as_string(mm) - - s = '\0' * 3 - buffer_support.zero_out_writebuffer(mm) - assert s == ''.join(mm) - assert s == buffer_support.readbuffer_as_string(mm) - assert s == buffer_support.writebuffer_as_string(mm) - assert s == buffer_support.charbuffer_as_string(mm) - - s = '\0' * 3 - ro_mm = mmap.mmap(-1, 3, access=mmap.ACCESS_READ) - assert buffer_support.check_readbuffer(ro_mm) - assert s == buffer_support.readbuffer_as_string(ro_mm) - assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm) - assert s == buffer_support.charbuffer_as_string(ro_mm) - - def test_array(self): - import array - buffer_support = self.get_buffer_support() - - s = 'a\0x' - a = array.array('B', [5, 0, 10]) - - buffer_support.zero_out_writebuffer(a) - assert list(a) == [0, 0, 0] - - def test_nonbuffer(self): - # e.g. int - buffer_support = self.get_buffer_support() - - assert not buffer_support.check_readbuffer(42) - assert raises(TypeError, buffer_support.readbuffer_as_string, 42) - assert raises(TypeError, buffer_support.writebuffer_as_string, 42) - assert raises(TypeError, buffer_support.charbuffer_as_string, 42) - - def test_user_class(self): - class MyBuf(str): - pass - s = 'a\0x' - buf = MyBuf(s) - buffer_support = self.get_buffer_support() - - assert buffer_support.check_readbuffer(buf) - assert s == buffer_support.readbuffer_as_string(buf) - assert raises(TypeError, buffer_support.writebuffer_as_string, buf) - assert s == buffer_support.charbuffer_as_string(buf) - - diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_bufferobject.py +++ /dev/null @@ -1,123 +0,0 @@ -from rpython.rtyper.lltypesystem import lltype -from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.api import PyObject - -class AppTestBufferObject(AppTestCpythonExtensionBase): - - def test_FromMemory(self): - module = self.import_extension('foo', [ - ("get_FromMemory", "METH_NOARGS", - """ - cbuf = malloc(4); - cbuf[0] = 'a'; - cbuf[1] = 'b'; - cbuf[2] = 'c'; - cbuf[3] = '\\0'; - return PyBuffer_FromMemory(cbuf, 4); - """), - ("free_buffer", "METH_NOARGS", - """ - free(cbuf); - Py_RETURN_NONE; - """), - ("check_ascharbuffer", "METH_O", - """ - char *ptr; - Py_ssize_t size; - if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0) - return NULL; - return PyString_FromStringAndSize(ptr, size); - """) - ], prologue = """ - static char* cbuf = NULL; - """) - buf = module.get_FromMemory() - assert str(buf) == 'abc\0' - - assert module.check_ascharbuffer(buf) == 'abc\0' - - module.free_buffer() - - def test_Buffer_New(self): - module = self.import_extension('foo', [ - ("buffer_new", "METH_NOARGS", - """ - return PyBuffer_New(150); - """), - ]) - b = module.buffer_new() - raises(AttributeError, getattr, b, 'x') - - def test_array_buffer(self): - if self.runappdirect: - skip('PyBufferObject not available outside buffer object.c') - module = self.import_extension('foo', [ - ("roundtrip", "METH_O", - """ - PyBufferObject *buf = (PyBufferObject *)args; - return PyString_FromStringAndSize(buf->b_ptr, buf->b_size); - """), - ]) - import array - a = array.array('c', 'text') - b = buffer(a) - assert module.roundtrip(b) == 'text' - - - def test_issue2752(self): - iterations = 10 - if self.runappdirect: - iterations = 2000 - module = self.import_extension('foo', [ - ("test_mod", 'METH_VARARGS', - """ - PyObject *obj; - Py_buffer bp; - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) - return NULL; - - if (((unsigned char*)bp.buf)[0] != '0') { - void * buf = (void*)bp.buf; - unsigned char val[4]; - char * s = PyString_AsString(obj); - memcpy(val, bp.buf, 4); - PyBuffer_Release(&bp); - if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) - return NULL; - PyErr_Format(PyExc_ValueError, - "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", - buf, val[0], val[1], val[2], val[3], - (void *)bp.buf, - ((unsigned char*)bp.buf)[0], - ((unsigned char*)bp.buf)[1], - ((unsigned char*)bp.buf)[2], - ((unsigned char*)bp.buf)[3], - s); - PyBuffer_Release(&bp); - return NULL; - } - - PyBuffer_Release(&bp); - Py_RETURN_NONE; - """), - ]) - bufsize = 4096 - def getdata(bufsize): - data = b'01234567' - for x in range(18): - data += data - if len(data) >= bufsize: - break - return data - for j in range(iterations): - block = getdata(bufsize) - assert block[:8] == '01234567' - try: - module.test_mod(block) - except ValueError as e: - print("%s at it=%d" % (e, j)) - assert False diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_intobject.py +++ /dev/null @@ -1,247 +0,0 @@ -from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.intobject import ( - PyInt_Check, PyInt_AsLong, PyInt_AS_LONG, - PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask, - PyInt_AsUnsignedLongLongMask) -from pypy.module.cpyext.pyobject import (decref, make_ref, - get_w_obj_and_decref) -from pypy.module.cpyext.state import State -import sys - -class TestIntObject(BaseApiTest): - def test_intobject(self, space): - state = space.fromcache(State) - assert PyInt_Check(space, space.wrap(3)) - assert PyInt_Check(space, space.w_True) - assert not PyInt_Check(space, space.wrap((1, 2, 3))) - for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]: - x = PyInt_AsLong(space, space.wrap(i)) - y = PyInt_AS_LONG(space, space.wrap(i)) - assert x == i - assert y == i - py_x = state.C.PyInt_FromLong(x + 1) - w_x = get_w_obj_and_decref(space, py_x) - assert space.type(w_x) is space.w_int - assert space.eq_w(w_x, space.wrap(i + 1)) - - with raises_w(space, TypeError): - PyInt_AsLong(space, space.w_None) - - with raises_w(space, TypeError): - PyInt_AsLong(space, None) - - assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint - with raises_w(space, ValueError): - PyInt_AsUnsignedLong(space, space.wrap(-5)) - - assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint)) - == sys.maxint) - assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30)) - == 10 ** 30 % ((sys.maxint + 1) * 2)) - - assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint)) - == sys.maxint) - assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30)) - == 10 ** 30 % (2 ** 64)) - - def test_freelist_direct(self, space): - state = space.fromcache(State) - p_x = state.C.PyInt_FromLong(12345678) - decref(space, p_x) - p_y = state.C.PyInt_FromLong(87654321) - # check that the address is the same, i.e. that the freelist did its - # job - assert p_x == p_y - decref(space, p_y) - - def test_freelist_make_ref(self, space): - w_x = space.newint(12345678) - w_y = space.newint(87654321) - p_x = make_ref(space, w_x) - decref(space, p_x) - p_y = make_ref(space, w_y) - # check that the address is the same: note that w_x does NOT keep p_x - # alive, because in make_ref we have a special case for ints - assert p_x == p_y - decref(space, p_y) - - def test_freelist_int_subclass(self, space): - w_MyInt = space.appexec([], """(): - class MyInt(int): - pass - return MyInt""") - w_x = space.call_function(w_MyInt, space.newint(12345678)) - w_y = space.call_function(w_MyInt, space.newint(87654321)) - p_x = make_ref(space, w_x) - decref(space, p_x) - p_y = make_ref(space, w_y) - # now the address is different because the freelist does not work for - # int subclasses - assert p_x != p_y - decref(space, p_y) - - def test_coerce(self, space): - w_obj = space.appexec([], """(): - class Coerce(object): - def __int__(self): - return 42 - return Coerce()""") - assert PyInt_AsLong(space, w_obj) == 42 - -class AppTestIntObject(AppTestCpythonExtensionBase): - def test_fromstring(self): - module = self.import_extension('foo', [ - ("from_string", "METH_NOARGS", - """ - return PyInt_FromString("1234", NULL, 16); - """), - ]) - assert module.from_string() == 0x1234 - assert type(module.from_string()) is int - - def test_size_t(self): - module = self.import_extension('foo', [ - ("values", "METH_NOARGS", - """ - return Py_BuildValue("NNNN", - PyInt_FromSize_t(123), - PyInt_FromSize_t((size_t)-1), - PyInt_FromSsize_t(123), - PyInt_FromSsize_t((size_t)-1)); - """), - ]) - values = module.values() - types = [type(x) for x in values] - assert types == [int, long, int, int] - - def test_int_subtype(self): - module = self.import_extension( - 'foo', [ - ("newEnum", "METH_VARARGS", - """ - EnumObject *enumObj; - int intval; - PyObject *name; - - if (!PyArg_ParseTuple(args, "Oi", &name, &intval)) - return NULL; - - enumObj = PyObject_New(EnumObject, &Enum_Type); - if (!enumObj) { - return NULL; - } - - enumObj->ob_ival = intval; - Py_INCREF(name); - enumObj->ob_name = name; - - return (PyObject *)enumObj; - """), - ], - prologue=""" - #include "structmember.h" - typedef struct - { - PyObject_HEAD - long ob_ival; - PyObject* ob_name; - } EnumObject; - - static void - enum_dealloc(PyObject *op) - { - Py_DECREF(((EnumObject *)op)->ob_name); - Py_TYPE(op)->tp_free(op); - } - - static PyMemberDef enum_members[] = { - {"name", T_OBJECT, offsetof(EnumObject, ob_name), 0, NULL}, - {NULL} /* Sentinel */ - }; - - PyTypeObject Enum_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - /*tp_name*/ "Enum", - /*tp_basicsize*/ sizeof(EnumObject), - /*tp_itemsize*/ 0, - /*tp_dealloc*/ enum_dealloc, - /*tp_print*/ 0, - /*tp_getattr*/ 0, - /*tp_setattr*/ 0, - /*tp_compare*/ 0, - /*tp_repr*/ 0, - /*tp_as_number*/ 0, - /*tp_as_sequence*/ 0, - /*tp_as_mapping*/ 0, - /*tp_hash*/ 0, - /*tp_call*/ 0, - /*tp_str*/ 0, - /*tp_getattro*/ 0, - /*tp_setattro*/ 0, - /*tp_as_buffer*/ 0, - /*tp_flags*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - /*tp_doc*/ 0, - /*tp_traverse*/ 0, - /*tp_clear*/ 0, - /*tp_richcompare*/ 0, - /*tp_weaklistoffset*/ 0, - /*tp_iter*/ 0, - /*tp_iternext*/ 0, - /*tp_methods*/ 0, - /*tp_members*/ enum_members, - /*tp_getset*/ 0, - /*tp_base*/ 0, /* set to &PyInt_Type in init function for MSVC */ - /*tp_dict*/ 0, - /*tp_descr_get*/ 0, - /*tp_descr_set*/ 0, - /*tp_dictoffset*/ 0, - /*tp_init*/ 0, - /*tp_alloc*/ 0, - /*tp_new*/ 0 - }; - """, more_init = ''' - Enum_Type.tp_base = &PyInt_Type; - if (PyType_Ready(&Enum_Type) < 0) INITERROR; - ''') - - a = module.newEnum("ULTIMATE_ANSWER", 42) - assert type(a).__name__ == "Enum" - assert isinstance(a, int) - assert a == int(a) == 42 - assert a.name == "ULTIMATE_ANSWER" - - def test_int_cast(self): - mod = self.import_extension('foo', [ - #prove it works for ints - ("test_int", "METH_NOARGS", - """ - PyObject * obj = PyInt_FromLong(42); - PyObject * val; - if (!PyInt_Check(obj)) { - Py_DECREF(obj); - PyErr_SetNone(PyExc_ValueError); - return NULL; - } - val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival); - Py_DECREF(obj); - return val; - """ - ), - ]) - i = mod.test_int() - assert isinstance(i, int) - assert i == 42 - - def test_int_macros(self): - mod = self.import_extension('foo', [ - ("test_macros", "METH_NOARGS", - """ - PyObject * obj = PyInt_FromLong(42); - PyIntObject * i = (PyIntObject*)obj; - PyInt_AS_LONG(obj); - PyInt_AS_LONG(i); - Py_RETURN_NONE; - """ - ), - ]) diff --git a/pypy/module/cpyext/test/test_pycobject.py b/pypy/module/cpyext/test/test_pycobject.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_pycobject.py +++ /dev/null @@ -1,30 +0,0 @@ -import py -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - -class AppTestStringObject(AppTestCpythonExtensionBase): - def test_pycobject_import(self): - module = self.import_extension('foo', [ - ("set_ptr", "METH_O", - """ - PyObject *pointer, *module; - void *ptr = PyLong_AsVoidPtr(args); - if (PyErr_Occurred()) return NULL; - pointer = PyCObject_FromVoidPtr(ptr, NULL); - if (PyErr_Occurred()) return NULL; - module = PyImport_ImportModule("foo"); - PyModule_AddObject(module, "_ptr", pointer); - Py_DECREF(module); - if (PyErr_Occurred()) return NULL; - Py_RETURN_NONE; - """), - ("get_ptr", "METH_NOARGS", - """ - void *ptr = PyCObject_Import("foo", "_ptr"); - if (PyErr_Occurred()) return NULL; - return PyLong_FromVoidPtr(ptr); - """)]) - module.set_ptr(1234) - assert "PyCObject object" in str(module._ptr) - import gc; gc.collect() - assert module.get_ptr() == 1234 - del module._ptr diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1247,7 +1247,7 @@ value = function(*args) assert isinstance(value, int) else: - if not value: + if not isinstance(value, list) and not value: value = function(*args) assert not isinstance(value, int) return value diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): From pypy.commits at gmail.com Thu Nov 8 13:45:44 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 08 Nov 2018 10:45:44 -0800 (PST) Subject: [pypy-commit] pypy py3tests: hg merge py3.5 Message-ID: <5be48458.1c69fb81.33d14.ea3f@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r95282:e018b8a5f031 Date: 2018-11-07 16:47 +0000 http://bitbucket.org/pypy/pypy/changeset/e018b8a5f031/ Log: hg merge py3.5 diff too long, truncating to 2000 out of 6925 lines diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Welcome to PyPy! -PyPy is an interperter that implements the Python programming language, based +PyPy is an interpreter that implements the Python programming language, based on the RPython compiler framework for dynamic language implementations. The home page for the interpreter is: @@ -15,29 +15,29 @@ http://doc.pypy.org/ -More documentation about the RPython framework can be found here +More documentation about the RPython framework can be found here: - http://rpython.readthedocs.io + http://rpython.readthedocs.io/ -The source for the documentation is in the pypy/doc directory +The source for the documentation is in the pypy/doc directory. + Using PyPy instead of CPython -============================= +----------------------------- -Please read the information at http://pypy.org to find the correct way to +Please read the information at http://pypy.org/ to find the correct way to download and use PyPy as an alternative to CPython. + Building -======== +-------- Building PyPy is not the recommended way to obtain the PyPy alternative python interpreter. It is time-consuming and requires significant computing resources. -More information can be found here +More information can be found here: http://doc.pypy.org/en/latest/build.html Enjoy and send us feedback! the pypy-dev team - - diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -8,11 +8,9 @@ available. """ -__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" - import sys import os -import imp +import imp, _imp from distutils.errors import DistutilsPlatformError @@ -71,9 +69,17 @@ def _init_nt(): """Initialize the module as appropriate for NT""" g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] # xxx? + g['VERSION'] = get_python_version().replace(".", "") + g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) global _config_vars _config_vars = g diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py --- a/lib-python/3/test/test_inspect.py +++ b/lib-python/3/test/test_inspect.py @@ -61,6 +61,9 @@ git = mod.StupidGit() +class ExampleClassWithSlot(object): + __slots__ = 'myslot' + class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, @@ -131,8 +134,11 @@ self.istest(inspect.iscoroutinefunction, 'coroutine_function_example') if hasattr(types, 'MemberDescriptorType'): - self.istest(inspect.ismemberdescriptor, - 'type(lambda: None).__globals__') + # App-level slots are member descriptors on both PyPy and + # CPython, but the various built-in attributes are all + # getsetdescriptors on PyPy. So check ismemberdescriptor() + # with an app-level slot. + self.istest(inspect.ismemberdescriptor, 'ExampleClassWithSlot.myslot') else: self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days)) diff --git a/lib-python/3/types.py b/lib-python/3/types.py --- a/lib-python/3/types.py +++ b/lib-python/3/types.py @@ -41,9 +41,19 @@ FrameType = type(tb.tb_frame) tb = None; del tb -# For Jython, the following two types are identical +# +# On CPython, FunctionType.__code__ is a 'getset_descriptor', but +# FunctionType.__globals__ is a 'member_descriptor', just like app-level +# slots. On PyPy, all descriptors of built-in types are +# 'getset_descriptor', but the app-level slots are 'member_descriptor' +# as well. (On Jython the situation might still be different.) +# +# Note that MemberDescriptorType was equal to GetSetDescriptorType in +# PyPy <= 6.0. +# GetSetDescriptorType = type(FunctionType.__code__) -MemberDescriptorType = type(FunctionType.__globals__) +class _C: __slots__ = 's' +MemberDescriptorType = type(_C.s) del sys, _f, _g, _C, _c, # Not for export diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -186,7 +186,7 @@ # Prefer poll, if available, since you can poll() any fd # which can't be done with select(). if HAVE_POLL: - p.register(sock.fileno(), POLLOUT | POLLIN) + p.register(sock.fileno(), POLLOUT if writing else POLLIN) rc = len(p.poll(timeout * 1000.0)) else: diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -189,6 +189,7 @@ self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg + _init_no_arg_ = __init__ def _fix_index(self, index): if index < 0: diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -110,7 +110,7 @@ raise ValueError( "Buffer size too small (%d instead of at least %d bytes)" % (buf.nbytes, offset + size)) - result = self() + result = self._newowninstance_() dest = result._buffer.buffer try: raw_addr = buf._pypy_raw_address() + offset @@ -121,6 +121,11 @@ memmove(dest, raw_addr, size) return result + def _newowninstance_(self): + result = self.__new__(self) + result._init_no_arg_() + return result + class CArgObject(object): """ simple wrapper around buffer, just for the case of freeing @@ -151,6 +156,7 @@ def __init__(self, *args, **kwds): raise TypeError("%s has no type" % (type(self),)) + _init_no_arg_ = __init__ def _ensure_objects(self): if '_objects' not in self.__dict__: diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -267,6 +267,7 @@ return raise TypeError("Unknown constructor %s" % (args,)) + _init_no_arg_ = __init__ def _wrap_callable(self, to_call, argtypes): def f(*args): @@ -485,6 +486,8 @@ return cobj, cobj._to_ffi_param(), type(cobj) def _convert_args_for_callback(self, argtypes, args): + from _ctypes.structure import StructOrUnion + # assert len(argtypes) == len(args) newargs = [] for argtype, arg in zip(argtypes, args): @@ -494,6 +497,10 @@ param = param._get_buffer_value() elif self._is_primitive(argtype): param = param.value + elif isinstance(param, StructOrUnion): # not a *pointer* to struct + newparam = StructOrUnion.__new__(type(param)) + param._copy_to(newparam._buffer.buffer) + param = newparam newargs.append(param) return newargs @@ -557,7 +564,7 @@ keepalive, newarg, newargtype = self._conv_param(argtype, defval) else: import ctypes - val = argtype._type_() + val = argtype._type_._newowninstance_() keepalive = None newarg = ctypes.byref(val) newargtype = type(newarg) diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -67,8 +67,11 @@ self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value + def _init_no_arg_(self): + self._buffer = ffiarray(1, autofree=True) self._ffiarray = ffiarray self.__init__ = __init__ + self._init_no_arg_ = _init_no_arg_ self._type_ = TP def _build_ffiargtype(self): @@ -136,27 +139,21 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) + result = tp._newowninstance_() if isinstance(obj, int): - result = tp() result._buffer[0] = obj return result elif obj is None: - result = tp() return result elif isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - result = ptr + result._buffer[0] = obj._buffer elif isinstance(obj, bytes): - result = tp() result._buffer[0] = memoryview(obj)._pypy_raw_address() return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) else: - result = tp() result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -378,11 +378,14 @@ self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value + _init_no_arg_ = __init__ def _ensure_objects(self): - if self._type_ not in 'zZP': - assert self._objects is None - return self._objects + # No '_objects' is the common case for primitives. Examples + # where there is an _objects is if _type in 'zZP', or if + # self comes from 'from_buffer(buf)'. See module/test_lib_pypy/ + # ctypes_test/test_buffers.py: test_from_buffer_keepalive. + return getattr(self, '_objects', None) def _getvalue(self): return self._buffer[0] diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -280,6 +280,7 @@ self.__setattr__(name, arg) for name, arg in kwds.items(): self.__setattr__(name, arg) + _init_no_arg_ = __init__ def _subarray(self, fieldtype, name): """Return a _rawffi array of length 1 whose address is the same as diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h --- a/lib_pypy/cffi/_cffi_errors.h +++ b/lib_pypy/cffi/_cffi_errors.h @@ -50,7 +50,9 @@ "import sys\n" "class FileLike:\n" " def write(self, x):\n" - " of.write(x)\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" " self.buf += x\n" "fl = FileLike()\n" "fl.buf = ''\n" diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,20 +8,43 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350 is still open: on Windows, the code here causes it to link - with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was - attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv - does not make PYTHON3.DLL available, and so the "correctly" compiled - version would not run inside a virtualenv. We will re-apply the fix - after virtualenv has been fixed for some time. For explanation, see - issue #355. For a workaround if you want PYTHON3.DLL and don't worry - about virtualenv, see issue #350. See also 'py_limited_api' in - setuptools_ext.py. + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need virtualenv version + >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround + you can remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -636,6 +636,10 @@ if isinstance(init, bytes): init = [init[i:i+1] for i in range(len(init))] else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) init = tuple(init) if len(init) > len(blob): raise IndexError("too many initializers") diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,13 +81,8 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) - - On Windows, it's better not to use py_limited_api until issue #355 - can be resolved (by having virtualenv copy PYTHON3.DLL). See also - the start of _cffi_include.h. """ - if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') - and sys.platform != 'win32'): + if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -167,6 +162,17 @@ module_path = module_name.split('.') module_path[-1] += '.py' generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules dist.cmdclass['build_py'] = build_py_make_mod # distutils and setuptools have no notion I could find of a @@ -176,6 +182,7 @@ # the module. So we add it here, which gives a few apparently # harmless warnings about not finding the file outside the # build directory. + # Then we need to hack more in get_source_files(); see above. if dist.py_modules is None: dist.py_modules = [] dist.py_modules.append(module_name) diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py --- a/lib_pypy/grp.py +++ b/lib_pypy/grp.py @@ -5,8 +5,8 @@ import os from _pwdgrp_cffi import ffi, lib import _structseq -import thread -_lock = thread.allocate_lock() +import _thread +_lock = _thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py --- a/lib_pypy/pwd.py +++ b/lib_pypy/pwd.py @@ -12,8 +12,8 @@ from _pwdgrp_cffi import ffi, lib import _structseq -import thread -_lock = thread.allocate_lock() +import _thread +_lock = _thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -27,6 +27,12 @@ from .console import Console, Event from .unix_eventqueue import EventQueue from .trace import trace +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -76,8 +82,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -407,6 +413,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError) as err: diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -200,6 +200,10 @@ default=False, requires=[("objspace.usemodules.cpyext", False)]), + BoolOption("disable_entrypoints_in_cffi", + "Disable only cffi's embedding mode.", + default=False), + BoolOption("fstrings", "if you are really convinced that f-strings are a security " "issue, you can disable them here", diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -330,7 +330,8 @@ ------------- * Hash randomization (``-R``) `is ignored in PyPy`_. In CPython - before 3.4 it has `little point`_. + before 3.4 it has `little point`_. Both CPython >= 3.4 and PyPy3 + implement the randomized SipHash algorithm and ignore ``-R``. * You can't store non-string keys in type objects. For example:: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -7,7 +7,8 @@ .. branch: cppyy-packaging -Upgrade to backend 1.2.0, improved handling of templated methods and +Main items: vastly better template resolution and improved performance. In +detail: upgrade to backend 1.4, improved handling of templated methods and functions (in particular automatic deduction of types), improved pythonization interface, range of compatibility fixes for Python3, free functions now take fast libffi path when possible, moves for strings (incl. from Python str), @@ -35,3 +36,18 @@ .. branch: pyparser-improvements-3 Small refactorings in the Python parser. + +.. branch: fix-readme-typo + +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1511,10 +1511,19 @@ if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE: raise oefmt(self.w_BufferError, "Object is not writable.") + def _try_buffer_w(self, w_obj, flags): + if not we_are_translated(): + if w_obj.buffer_w.im_func != W_Root.buffer_w.im_func: + # when 'buffer_w()' is overridden in the subclass of + # W_Root, we need to specify __buffer="read" or + # __buffer="read-write" in the TypeDef. + assert type(w_obj).typedef.buffer is not None + return w_obj.buffer_w(self, flags) + def buffer_w(self, w_obj, flags): # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer) try: - return w_obj.buffer_w(self, flags) + return self._try_buffer_w(w_obj, flags) except BufferInterfaceNotFound: raise oefmt(self.w_TypeError, "'%T' does not support the buffer interface", w_obj) @@ -1522,14 +1531,14 @@ def readbuf_w(self, w_obj): # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer) try: - return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf() + return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf() except BufferInterfaceNotFound: self._getarg_error("bytes-like object", w_obj) def writebuf_w(self, w_obj): # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer) try: - return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf() + return self._try_buffer_w(w_obj, self.BUF_WRITABLE).as_writebuf() except (BufferInterfaceNotFound, OperationError): self._getarg_error("read-write bytes-like object", w_obj) @@ -1563,7 +1572,7 @@ # NB. CPython forbids surrogates here return StringBuffer(w_obj.text_w(self)) try: - return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf() + return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf() except BufferInterfaceNotFound: self._getarg_error("bytes or buffer", w_obj) elif code == 's#': @@ -1575,7 +1584,7 @@ if self.isinstance_w(w_obj, self.w_unicode): # NB. CPython forbids return w_obj.text_w(self) # surrogates here try: - return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str() + return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_str() except BufferInterfaceNotFound: self._getarg_error("bytes or read-only buffer", w_obj) elif code == 'w*': diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -2076,7 +2076,7 @@ else: skip_leading_underscores = False for name in all: - if skip_leading_underscores and name[0]=='_': + if skip_leading_underscores and name and name[0] == '_': continue into_locals[name] = getattr(module, name) ''', filename=__file__) diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -236,7 +236,7 @@ next_token_seen is not None and next_token_seen.value != '('): msg = "Missing parentheses in call to '%s'" % ( - last_token_seen,) + last_token_seen.value,) else: msg = "invalid syntax" if e.expected_str is not None: diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -701,6 +701,15 @@ pass """ + def test_bug_annotation_inside_nested_function(self): + """ + # this used to crash + def f1(): + def f2(*args: int): + pass + f1() + """ + class AppTestSyntaxError: def test_tokenizer_error_location(self): diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -28,6 +28,9 @@ self.bases = bases # Used in cpyext to fill tp_as_buffer slots assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer" + for base in bases: + if __buffer is None: + __buffer = base.buffer self.buffer = __buffer self.heaptype = False self.hasdict = '__dict__' in rawdict diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -108,97 +108,99 @@ max_jitdriver = jit.JitDriver(name='max', greens=['has_key', 'has_item', 'w_type'], reds='auto') -def make_min_max(unroll): - @specialize.arg(2) - def min_max_impl(space, args, implementation_of): - if implementation_of == "max": - compare = space.gt - jitdriver = max_jitdriver + at specialize.arg(4) +def min_max_sequence(space, w_sequence, w_key, w_default, implementation_of): + if implementation_of == "max": + compare = space.gt + jitdriver = max_jitdriver + else: + compare = space.lt + jitdriver = min_jitdriver + w_iter = space.iter(w_sequence) + w_type = space.type(w_iter) + has_key = w_key is not None + has_item = False + w_max_item = w_default + w_max_val = None + while True: + jitdriver.jit_merge_point(has_key=has_key, has_item=has_item, + w_type=w_type) + try: + w_item = space.next(w_iter) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + break + if has_key: + w_compare_with = space.call_function(w_key, w_item) else: - compare = space.lt - jitdriver = min_jitdriver - any_kwds = bool(args.keywords) - args_w = args.arguments_w - if len(args_w) > 1: - if unroll and len(args_w) == 2 and not any_kwds: - # a fast path for the common case, useful for interpreted - # mode and to reduce the length of the jit trace - w0, w1 = args_w - if space.is_true(compare(w1, w0)): - return w1 - else: - return w0 - w_sequence = space.newtuple(args_w) - elif len(args_w): - w_sequence = args_w[0] + w_compare_with = w_item + if (not has_item or + space.is_true(compare(w_compare_with, w_max_val))): + has_item = True + w_max_item = w_item + w_max_val = w_compare_with + if w_max_item is None: + raise oefmt(space.w_ValueError, "arg is an empty sequence") + return w_max_item + + at specialize.arg(3) + at jit.look_inside_iff(lambda space, args_w, w_key, implementation_of: + jit.loop_unrolling_heuristic(args_w, len(args_w), 3)) +def min_max_multiple_args(space, args_w, w_key, implementation_of): + # case of multiple arguments (at least two). We unroll it if there + # are 2 or 3 arguments. + if implementation_of == "max": + compare = space.gt + else: + compare = space.lt + w_max_item = args_w[0] + if w_key is not None: + w_max_val = space.call_function(w_key, w_max_item) + else: + w_max_val = w_max_item + for i in range(1, len(args_w)): + w_item = args_w[i] + if w_key is not None: + w_compare_with = space.call_function(w_key, w_item) else: - raise oefmt(space.w_TypeError, - "%s() expects at least one argument", - implementation_of) - w_key = None - w_default = None - if any_kwds: - kwds = args.keywords - for n in range(len(kwds)): - if kwds[n] == "key": - w_key = args.keywords_w[n] - elif kwds[n] == "default": - w_default = args.keywords_w[n] - else: - raise oefmt(space.w_TypeError, - "%s() got unexpected keyword argument", - implementation_of) + w_compare_with = w_item + if space.is_true(compare(w_compare_with, w_max_val)): + w_max_item = w_item + w_max_val = w_compare_with + return w_max_item - if w_default is not None and len(args_w) > 1: - raise oefmt(space.w_TypeError, - "Cannot specify a default for %s() with multiple positional arguments", - implementation_of) - - w_iter = space.iter(w_sequence) - w_type = space.type(w_iter) - has_key = w_key is not None - has_item = False - w_max_item = None - w_max_val = None - while True: - if not unroll: - jitdriver.jit_merge_point(has_key=has_key, has_item=has_item, w_type=w_type) - try: - w_item = space.next(w_iter) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - break - if has_key: - w_compare_with = space.call_function(w_key, w_item) - else: - w_compare_with = w_item - if not has_item or \ - space.is_true(compare(w_compare_with, w_max_val)): - has_item = True - w_max_item = w_item - w_max_val = w_compare_with - if w_max_item is None: - if w_default is not None: - w_max_item = w_default - else: - raise oefmt(space.w_ValueError, "arg is an empty sequence") - return w_max_item - if unroll: - min_max_impl = jit.unroll_safe(min_max_impl) - return min_max_impl - -min_max_unroll = make_min_max(True) -min_max_normal = make_min_max(False) - + at jit.unroll_safe # the loop over kwds @specialize.arg(2) def min_max(space, args, implementation_of): - if not jit.we_are_jitted() or len(args.arguments_w) != 1 and \ - jit.loop_unrolling_heuristic(args.arguments_w, len(args.arguments_w)): - return min_max_unroll(space, args, implementation_of) + w_key = None + w_default = None + if bool(args.keywords): + kwds = args.keywords + for n in range(len(kwds)): + if kwds[n] == "key": + w_key = args.keywords_w[n] + elif kwds[n] == "default": + w_default = args.keywords_w[n] + else: + raise oefmt(space.w_TypeError, + "%s() got unexpected keyword argument", + implementation_of) + # + args_w = args.arguments_w + if len(args_w) > 1: + if w_default is not None: + raise oefmt(space.w_TypeError, + "Cannot specify a default for %s() with multiple " + "positional arguments", implementation_of) + return min_max_multiple_args(space, args_w, w_key, implementation_of) + elif len(args_w): + return min_max_sequence(space, args_w[0], w_key, w_default, + implementation_of) else: - return min_max_normal(space, args, implementation_of) -min_max._always_inline = True + raise oefmt(space.w_TypeError, + "%s() expects at least one argument", + implementation_of) def max(space, __args__): """max(iterable[, key=func]) -> value diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -654,6 +654,10 @@ raises(TypeError, max, 1, default=0) raises(TypeError, max, default=1) + def test_max_list_and_key(self): + assert max(["100", "50", "30", "-200"], key=int) == "100" + assert max("100", "50", "30", "-200", key=int) == "100" + try: from hypothesis import given, strategies, example diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -209,3 +209,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/__pypy__/moduledef.py b/pypy/module/__pypy__/moduledef.py --- a/pypy/module/__pypy__/moduledef.py +++ b/pypy/module/__pypy__/moduledef.py @@ -110,6 +110,7 @@ 'stack_almost_full' : 'interp_magic.stack_almost_full', 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } submodules = { diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -156,7 +156,7 @@ return MiniBuffer(LLBuffer(ptr, size), w_cdata) MiniBuffer.typedef = TypeDef( - "_cffi_backend.buffer", + "_cffi_backend.buffer", None, None, "read-write", __new__ = interp2app(MiniBuffer___new__), __len__ = interp2app(MiniBuffer.descr_len), __getitem__ = interp2app(MiniBuffer.descr_getitem), diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -79,13 +79,6 @@ w_result = self.ctype.cast_to_int(ptr) return w_result - def long(self, space): - w_result = self.int(space) - space = self.space - if space.is_w(space.type(w_result), space.w_int): - w_result = space.newlong(space.int_w(w_result)) - return w_result - def float(self): with self as ptr: w_result = self.ctype.float(ptr) @@ -664,7 +657,6 @@ __repr__ = interp2app(W_CData.repr), __bool__ = interp2app(W_CData.bool), __int__ = interp2app(W_CData.int), - __long__ = interp2app(W_CData.long), __float__ = interp2app(W_CData.float), __complex__ = interp2app(W_CData.complex), __len__ = interp2app(W_CData.len), diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -104,7 +104,15 @@ return self.ctptr def convert_from_object(self, cdata, w_ob): - self.convert_array_from_object(cdata, w_ob) + if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self: + length = w_ob.get_array_length() + with w_ob as source: + source = rffi.cast(rffi.VOIDP, source) + target = rffi.cast(rffi.VOIDP, cdata) + size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length) + rffi.c_memcpy(target, source, size) + else: + self.convert_array_from_object(cdata, w_ob) def convert_to_object(self, cdata): if self.length < 0: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -51,9 +51,12 @@ def unpack_list_of_float_items(self, ptr, length): return None - def pack_list_of_items(self, cdata, w_ob): + def pack_list_of_items(self, cdata, w_ob, expected_length): return False + def _within_bounds(self, actual_length, expected_length): + return expected_length < 0 or actual_length <= expected_length + def newp(self, w_init, allocator): space = self.space raise oefmt(space.w_TypeError, @@ -102,6 +105,11 @@ # ctype 'A' must be a pointer to same type, not cdata # 'B'", but with A=B, then give instead a different error # message to try to clear up the confusion + if self is w_got.ctype: + raise oefmt(space.w_SystemError, + "initializer for ctype '%s' is correct, but we get " + "an internal mismatch--please report a bug", + self.name) return oefmt(space.w_TypeError, "initializer for ctype '%s' appears indeed to " "be '%s', but the types are different (check " diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -292,9 +292,10 @@ return res return None - def pack_list_of_items(self, cdata, w_ob): + def pack_list_of_items(self, cdata, w_ob, expected_length): int_list = self.space.listview_int(w_ob) - if int_list is not None: + if (int_list is not None and + self._within_bounds(len(int_list), expected_length)): if self.size == rffi.sizeof(rffi.LONG): # fastest path from rpython.rlib.rrawarray import copy_list_to_raw_array cdata = rffi.cast(rffi.LONGP, cdata) @@ -305,7 +306,8 @@ if overflowed != 0: self._overflow(self.space.newint(overflowed)) return True - return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob) + return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob, + expected_length) class W_CTypePrimitiveUnsigned(W_CTypePrimitive): @@ -375,15 +377,17 @@ return res return None - def pack_list_of_items(self, cdata, w_ob): + def pack_list_of_items(self, cdata, w_ob, expected_length): int_list = self.space.listview_int(w_ob) - if int_list is not None: + if (int_list is not None and + self._within_bounds(len(int_list), expected_length)): overflowed = misc.pack_list_to_raw_array_bounds_unsigned( int_list, cdata, self.size, self.vrangemax) if overflowed != 0: self._overflow(self.space.newint(overflowed)) return True - return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob) + return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob, + expected_length) class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned): @@ -471,9 +475,10 @@ return res return None - def pack_list_of_items(self, cdata, w_ob): + def pack_list_of_items(self, cdata, w_ob, expected_length): float_list = self.space.listview_float(w_ob) - if float_list is not None: + if (float_list is not None and + self._within_bounds(len(float_list), expected_length)): if self.size == rffi.sizeof(rffi.DOUBLE): # fastest path from rpython.rlib.rrawarray import copy_list_to_raw_array cdata = rffi.cast(rffi.DOUBLEP, cdata) @@ -483,7 +488,8 @@ misc.pack_float_list_to_raw_array(float_list, cdata, rffi.FLOAT, rffi.FLOATP) return True - return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob) + return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob, + expected_length) def unpack_ptr(self, w_ctypeptr, ptr, length): result = self.unpack_list_of_float_items(ptr, length) @@ -553,13 +559,15 @@ # 'list(array-of-longdouble)' returns a list of cdata objects, # not a list of floats. - def pack_list_of_items(self, cdata, w_ob): + def pack_list_of_items(self, cdata, w_ob, expected_length): float_list = self.space.listview_float(w_ob) - if float_list is not None: + if (float_list is not None and + self._within_bounds(len(float_list), expected_length)): misc.pack_float_list_to_raw_array(float_list, cdata, rffi.LONGDOUBLE, rffi.LONGDOUBLEP) return True - return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob) + return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob, + expected_length) @jit.dont_look_inside def nonzero(self, cdata): diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -58,7 +58,7 @@ def _convert_array_from_listview(self, cdata, lst_w): space = self.space - if self.length >= 0 and len(lst_w) > self.length: + if not self._within_bounds(len(lst_w), self.length): raise oefmt(space.w_IndexError, "too many initializers for '%s' (got %d)", self.name, len(lst_w)) @@ -71,8 +71,8 @@ space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): - if self.ctitem.pack_list_of_items(cdata, w_ob): # fast path - pass + if self.ctitem.pack_list_of_items(cdata, w_ob, self.length): + pass # fast path else: self._convert_array_from_listview(cdata, space.listview(w_ob)) elif self.accept_str: @@ -317,9 +317,7 @@ if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool): self._must_be_string_of_zero_or_one(value) keepalives[i] = value - buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value) - rffi.cast(rffi.CCHARPP, cdata)[0] = buf - return ord(buf_flag) # 4, 5 or 6 + return misc.write_string_as_charp(cdata, value) # if (space.isinstance_w(w_init, space.w_list) or space.isinstance_w(w_init, space.w_tuple)): diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py --- a/pypy/module/_cffi_backend/errorbox.py +++ b/pypy/module/_cffi_backend/errorbox.py @@ -69,7 +69,10 @@ import sys class FileLike: def write(self, x): - of.write(x) + try: + of.write(x) + except: + pass self.buf += x fl = FileLike() fl.buf = '' diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -573,8 +573,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -585,7 +585,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,24 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError +from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -102,6 +112,12 @@ def write_raw_longdouble_data(target, source): rffi.cast(rffi.LONGDOUBLEP, target)[0] = source + at jit.dont_look_inside # lets get_nonmovingbuffer_final_null be inlined +def write_string_as_charp(target, string): + buf, buf_flag = rffi.get_nonmovingbuffer_final_null(string) + rffi.cast(rffi.CCHARPP, target)[0] = buf + return ord(buf_flag) # 4, 5 or 6 + # ____________________________________________________________ sprintf_longdouble = rffi.llexternal( @@ -129,21 +145,14 @@ # This version accepts a Python int too, and does convertions from # other types of objects. It refuses floats. try: - value = space.int_w(w_ob) + return space.int_w(w_ob, allow_conversion=False) except OperationError as e: if not (e.match(space, space.w_OverflowError) or e.match(space, space.w_TypeError)): raise - else: - return value - try: - bigint = space.bigint_w(w_ob, allow_conversion=False) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise if _is_a_float(space, w_ob): raise - bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) + bigint = space.bigint_w(w_ob, allow_conversion=True) try: return bigint.tolonglong() except OverflowError: @@ -151,20 +160,15 @@ def as_long(space, w_ob): # Same as as_long_long(), but returning an int instead. - if space.is_w(space.type(w_ob), space.w_int): # shortcut - return space.int_w(w_ob) try: - bigint = space.bigint_w(w_ob, allow_conversion=False) + return space.int_w(w_ob, allow_conversion=False) except OperationError as e: - if not e.match(space, space.w_TypeError): + if not (e.match(space, space.w_OverflowError) or + e.match(space, space.w_TypeError)): raise if _is_a_float(space, w_ob): raise - bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) - try: - return bigint.toint() - except OverflowError: - raise OperationError(space.w_OverflowError, space.newtext(ovf_msg)) + return space.int_w(w_ob, allow_conversion=True) def as_unsigned_long_long(space, w_ob, strict): # (possibly) convert and cast a Python object to an unsigned long long. @@ -172,23 +176,19 @@ # objects. If 'strict', complains with OverflowError; if 'not strict', # mask the result and round floats. try: - value = space.int_w(w_ob) + value = space.int_w(w_ob, allow_conversion=False) except OperationError as e: if not (e.match(space, space.w_OverflowError) or e.match(space, space.w_TypeError)): raise + if strict and _is_a_float(space, w_ob): + raise else: if strict and value < 0: raise OperationError(space.w_OverflowError, space.newtext(neg_msg)) return r_ulonglong(value) - try: - bigint = space.bigint_w(w_ob, allow_conversion=False) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - if strict and _is_a_float(space, w_ob): - raise - bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) + # note that if not 'strict', then space.int() will round down floats + bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) if strict: try: return bigint.toulonglong() @@ -202,13 +202,22 @@ def as_unsigned_long(space, w_ob, strict): # same as as_unsigned_long_long(), but returning just an Unsigned try: - bigint = space.bigint_w(w_ob, allow_conversion=False) + value = space.int_w(w_ob, allow_conversion=False) except OperationError as e: - if not e.match(space, space.w_TypeError): + if not (e.match(space, space.w_OverflowError) or + e.match(space, space.w_TypeError)): raise if strict and _is_a_float(space, w_ob): raise - bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) + else: + if strict and value < 0: + raise OperationError(space.w_OverflowError, space.newtext(neg_msg)) + if not we_are_translated(): + if isinstance(value, NonConstant): # hack for test_ztranslation + return r_uint(0) + return r_uint(value) + # note that if not 'strict', then space.int() will round down floats + bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) if strict: try: return bigint.touint() @@ -241,7 +250,12 @@ def _standard_object_as_bool(space, w_ob): if space.isinstance_w(w_ob, space.w_int): - return space.bigint_w(w_ob).tobool() + try: + return space.int_w(w_ob) != 0 + except OperationError as e: + if not e.match(space, space.w_OverflowError): + raise + return space.bigint_w(w_ob).tobool() if space.isinstance_w(w_ob, space.w_float): return space.float_w(w_ob) != 0.0 raise _NotStandardObject @@ -388,3 +402,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/moduledef.py b/pypy/module/_cffi_backend/moduledef.py --- a/pypy/module/_cffi_backend/moduledef.py +++ b/pypy/module/_cffi_backend/moduledef.py @@ -71,7 +71,8 @@ def __init__(self, space, *args): MixedModule.__init__(self, space, *args) # - if not space.config.objspace.disable_entrypoints: + if (not space.config.objspace.disable_entrypoints and + not space.config.objspace.disable_entrypoints_in_cffi): # import 'embedding', which has the side-effect of registering # the 'pypy_init_embedded_cffi_module' entry point from pypy.module._cffi_backend import embedding diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -334,8 +334,18 @@ max = (1 << (8*size-1)) - 1 assert newp(pp, min)[0] == min assert newp(pp, max)[0] == max + py.test.raises(OverflowError, newp, pp, min - 2 ** 32) + py.test.raises(OverflowError, newp, pp, min - 2 ** 64) + py.test.raises(OverflowError, newp, pp, max + 2 ** 32) + py.test.raises(OverflowError, newp, pp, max + 2 ** 64) py.test.raises(OverflowError, newp, pp, min - 1) py.test.raises(OverflowError, newp, pp, max + 1) + py.test.raises(OverflowError, newp, pp, min - 1 - 2 ** 32) + py.test.raises(OverflowError, newp, pp, min - 1 - 2 ** 64) + py.test.raises(OverflowError, newp, pp, max + 1) + py.test.raises(OverflowError, newp, pp, max + 1 + 2 ** 32) + py.test.raises(OverflowError, newp, pp, max + 1 + 2 ** 64) + py.test.raises(TypeError, newp, pp, 1.0) for name in ['char', 'short', 'int', 'long', 'long long']: p = new_primitive_type('unsigned ' + name) pp = new_pointer_type(p) @@ -1862,7 +1872,7 @@ def test_newp_copying(): """Test that we can do newp(, ) for most - types, with the exception of arrays, like in C. + types, including same-type arrays. """ BInt = new_primitive_type("int") p = newp(new_pointer_type(BInt), cast(BInt, 42)) @@ -1891,8 +1901,9 @@ a1 = newp(BArray, [1, 2, 3, 4]) py.test.raises(TypeError, newp, BArray, a1) BArray6 = new_array_type(new_pointer_type(BInt), 6) - a1 = newp(BArray6, None) - py.test.raises(TypeError, newp, BArray6, a1) + a1 = newp(BArray6, [10, 20, 30]) + a2 = newp(BArray6, a1) + assert list(a2) == [10, 20, 30, 0, 0, 0] # s1 = newp(BStructPtr, [42]) s2 = newp(BStructPtr, s1[0]) @@ -3946,6 +3957,7 @@ z3 = cast(BVoidP, 0) z4 = cast(BUCharP, 0) with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") newp(new_pointer_type(BIntP), z1) # warn assert len(w) == 1 newp(new_pointer_type(BVoidP), z1) # fine diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py --- a/pypy/module/_cffi_backend/test/test_fastpath.py +++ b/pypy/module/_cffi_backend/test/test_fastpath.py @@ -267,3 +267,17 @@ assert lst == [1.25, -2.5, 3.75] if not self.runappdirect: assert self.get_count() == 1 + + def test_too_many_initializers(self): + import _cffi_backend + ffi = _cffi_backend.FFI() + raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "int[4]", tuple(range(999))) + raises(IndexError, ffi.new, "unsigned int[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "float[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "long double[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "char[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "wchar_t[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "_Bool[4]", [10, 20, 30, 40, 50]) + raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6]] * 5) + raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6,7]] * 4) diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -572,3 +572,13 @@ assert len(z) == 2 assert ffi.cast("int *", z)[0] == 0x12345 assert list(z) == [u'\U00012345', u'\x00'] # maybe a 2-unichars str + + def test_ffi_array_as_init(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + p = ffi.new("int[4]", [10, 20, 30, 400]) + q = ffi.new("int[4]", p) + assert list(q) == [10, 20, 30, 400] + raises(TypeError, ffi.new, "int[3]", p) + raises(TypeError, ffi.new, "int[5]", p) + raises(TypeError, ffi.new, "int16_t[4]", p) diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,16 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend, sys + sys.pypy_initfsencoding() # initialize space.sys.filesystemencoding + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -761,6 +761,26 @@ assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -1,13 +1,18 @@ import os + from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.rarithmetic import intmask from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel from rpython.rlib.rarithmetic import r_singlefloat from rpython.tool import leakfinder -from pypy.interpreter.gateway import interp2app -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.argument import Arguments +from pypy.interpreter.gateway import interp2app, interpindirect2app +from pypy.interpreter.typedef import TypeDef +from pypy.objspace.std.iterobject import W_AbstractSeqIterObject +from pypy.module._rawffi.array import W_ArrayInstance from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc from pypy.module._cffi_backend import newtype from pypy.module._cppyy import ffitypes @@ -23,10 +28,11 @@ class _Arg: # poor man's union _immutable_ = True - def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)): + def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)): self.tc = tc self._handle = h self._long = l + self._double = d self._string = s self._voidp = p @@ -40,6 +46,11 @@ def __init__(self, val): _Arg.__init__(self, 'l', l = val) +class _ArgD(_Arg): + _immutable_ = True + def __init__(self, val): + _Arg.__init__(self, 'd', d = val) + class _ArgS(_Arg): _immutable_ = True def __init__(self, val): @@ -89,6 +100,9 @@ assert obj._voidp != rffi.cast(rffi.VOIDP, 0) data = rffi.cast(rffi.VOIDPP, data) data[0] = obj._voidp + elif obj.tc == 'd': + assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat) + misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size) else: # only other use is string assert obj.tc == 's' n = len(obj._string) @@ -182,6 +196,7 @@ 'call_f' : ([c_method, c_object, c_int, c_voidp], c_float), 'call_d' : ([c_method, c_object, c_int, c_voidp], c_double), 'call_ld' : ([c_method, c_object, c_int, c_voidp], c_ldouble), + 'call_nld' : ([c_method, c_object, c_int, c_voidp], c_double), 'call_r' : ([c_method, c_object, c_int, c_voidp], c_voidp), # call_s actually takes an size_t* as last parameter, but this will do @@ -236,6 +251,8 @@ 'method_prototype' : ([c_scope, c_method, c_int], c_ccharp), 'is_const_method' : ([c_method], c_int), + 'get_num_templated_methods': ([c_scope], c_int), + 'get_templated_method_name': ([c_scope, c_index], c_ccharp), 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'get_method_template' : ([c_scope, c_ccharp, c_ccharp], c_method), @@ -272,9 +289,11 @@ 'stdstring2charp' : ([c_object, c_voidp], c_ccharp), 'stdstring2stdstring' : ([c_object], c_object), - 'stdvector_valuetype' : ([c_ccharp], c_ccharp), - 'stdvector_valuesize' : ([c_ccharp], c_size_t), + 'longdouble2double' : ([c_voidp], c_double), + 'double2longdouble' : ([c_double, c_voidp], c_void), + 'vectorbool_getitem' : ([c_object, c_int], c_int), + 'vectorbool_setitem' : ([c_object, c_int, c_int], c_void), } # size/offset are backend-specific but fixed after load @@ -289,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants @@ -401,7 +420,9 @@ return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args))) def c_call_ld(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] - return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args))) + #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args))) + # call_nld narrows long double to double + return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args))) def c_call_r(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] @@ -561,16 +582,21 @@ def c_is_const_method(space, cppmeth): return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)])) +def c_get_num_templated_methods(space, cppscope): + return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)])) +def c_get_templated_method_name(space, cppscope, index): + args = [_ArgH(cppscope.handle), _ArgL(index)] + return charp2str_free(space, call_capi(space, 'method_is_template', args)) def c_exists_method_template(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) - def c_get_method_template(space, cppscope, name, proto): args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)] return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args))) + def c_get_global_operator(space, nss, lc, rc, op): if nss is not None: args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)] @@ -619,7 +645,7 @@ return space.bool_w(call_capi(space, 'is_enum_data', args)) def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] - return space.bool_w(call_capi(space, 'get_dimension_size', args)) + return space.int_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -650,24 +676,94 @@ def c_stdstring2stdstring(space, cppobject): return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)])) -def c_stdvector_valuetype(space, pystr): - return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)])) +def c_longdouble2double(space, addr): + return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)])) +def c_double2longdouble(space, dval, addr): + call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)]) -def c_stdvector_valuetype(space, pystr): - return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)])) -def c_stdvector_valuesize(space, pystr): - return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)])) +def c_vectorbool_getitem(space, vbool, idx): + return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)]) +def c_vectorbool_setitem(space, vbool, idx, value): + call_capi(space, 'vectorbool_setitem', [_ArgH(vbool), _ArgL(idx), _ArgL(value)]) # TODO: factor these out ... # pythonizations def stdstring_c_str(space, w_self): """Return a python string taking into account \0""" - from pypy.module._cppyy import interp_cppyy cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) +def vbool_getindex(space, w_vbool, w_idx): + idx = space.getindex_w(w_idx, space.w_IndexError, "std::vector index") + sz = space.len_w(w_vbool) + if idx < 0: idx += sz + if idx < 0 or idx >= sz: + raise IndexError + return idx + +def vectorbool_getitem(space, w_self, w_idx): + """Index a std::vector, return the value""" + from pypy.module._cppyy import interp_cppyy + vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) + idx = vbool_getindex(space, w_self, w_idx) + item = c_vectorbool_getitem(space, vbool._rawobject, idx) + return space.newbool(space.is_true(item)) + +def vectorbool_setitem(space, w_self, w_idx, w_value): + """Index a std::vector, set the value""" + from pypy.module._cppyy import interp_cppyy + vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) + idx = vbool_getindex(space, w_self, w_idx) + c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value))) + +class W_STLVectorIter(W_AbstractSeqIterObject): + # w_seq and index are in base class + _immutable_fields_ = ['converter', 'data', 'len', 'stride'] + + def __init__(self, space, w_vector): + W_AbstractSeqIterObject.__init__(self, w_vector) + # TODO: this should live in rpythonize.py or something so that the + # imports can move to the top w/o getting circles + from pypy.module._cppyy import interp_cppyy + assert isinstance(w_vector, interp_cppyy.W_CPPInstance) + vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector) + + v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type') + v_size = c_size_of_type(space, v_type) + + if not v_type or not v_size: + raise NotImplementedError # fallback on getitem + + from pypy.module._cppyy import converter + self.converter = converter.get_converter(space, v_type, '') + + # this 'data' is from the decl, so not the pythonized data from pythonify.py + w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, [])) + arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True) + if not arr: + raise OperationError(space.w_StopIteration, space.w_None) + + self.data = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space))) + self.len = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, []))) + self.stride = v_size + + def descr_next(self, space): + if self.w_seq is None: + raise OperationError(space.w_StopIteration, space.w_None) + if self.len <= self.index: + self.w_seq = None + raise OperationError(space.w_StopIteration, space.w_None) + offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride)) + w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset)) + self.index += 1 + return w_item + +def stdvector_iter(space, w_self): + return W_STLVectorIter(space, w_self) + + # setup pythonizations for later use at run-time _pythonizations = {} def register_pythonizations(space): @@ -678,6 +774,12 @@ ### std::string stdstring_c_str, + ### std::vector + stdvector_iter, + + ### std::vector + vectorbool_getitem, + vectorbool_setitem, ] for f in allfuncs: @@ -692,3 +794,10 @@ space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"]) _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str") _method_alias(space, w_pycppclass, "__str__", "c_str") + + if name.find("std::vector Author: Ronan Lamy Branch: apptest-file Changeset: r95289:b41d84cd81ea Date: 2018-11-10 14:51 +0000 http://bitbucket.org/pypy/pypy/changeset/b41d84cd81ea/ Log: fix import order issues in cpyext tests diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -7,7 +7,6 @@ from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, from_ref, incref, decref, get_typedescr, hack_for_result_often_existing_obj) -from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError, oefmt @@ -264,7 +263,7 @@ if opid == Py_EQ: return 1 if opid == Py_NE: - return 0 + return 0 w_res = PyObject_RichCompare(space, w_o1, w_o2, opid_int) return int(space.is_true(w_res)) diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -1,5 +1,4 @@ from pypy.interpreter.function import Function -import pypy.module.cpyext.moduledef # XXX: avoids a weird import cycle issue from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.classobject import ( diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -14,6 +14,8 @@ from rpython.rlib import rawrefcount from rpython.tool.udir import udir +import pypy.module.cpyext.moduledef # Make sure all the functions are registered + only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) From pypy.commits at gmail.com Sat Nov 10 10:31:19 2018 From: pypy.commits at gmail.com (rlamy) Date: Sat, 10 Nov 2018 07:31:19 -0800 (PST) Subject: [pypy-commit] pypy apptest-file: fix MixedModule tests Message-ID: <5be6f9c7.1c69fb81.3e148.9e77@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r95290:e50107f99102 Date: 2018-11-10 15:30 +0000 http://bitbucket.org/pypy/pypy/changeset/e50107f99102/ Log: fix MixedModule tests diff --git a/pypy/interpreter/test/demomixedmod/__init__.py b/pypy/interpreter/test/demomixedmod/__init__.py --- a/pypy/interpreter/test/demomixedmod/__init__.py +++ b/pypy/interpreter/test/demomixedmod/__init__.py @@ -1,15 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - interpleveldefs = { - '__name__' : '(space.wrap("mixedmodule"))', - '__doc__' : '(space.wrap("mixedmodule doc"))', - 'somefunc' : 'file1.somefunc', - 'value' : '(space.w_None)', - 'path' : 'file1.initpath(space)', - 'cpypath' : 'space.wrap(sys.path)' - } - - appleveldefs = { - 'someappfunc' : 'file2_app.someappfunc', - } diff --git a/pypy/interpreter/test/demomixedmod/__init__.py b/pypy/interpreter/test/demomixedmod/moduledef.py copy from pypy/interpreter/test/demomixedmod/__init__.py copy to pypy/interpreter/test/demomixedmod/moduledef.py diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py --- a/pypy/interpreter/test/test_appinterp.py +++ b/pypy/interpreter/test/test_appinterp.py @@ -3,32 +3,32 @@ from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp from pypy.interpreter.error import OperationError -def test_execwith_novars(space): - val = space.appexec([], """ - (): - return 42 - """) +def test_execwith_novars(space): + val = space.appexec([], """ + (): + return 42 + """) assert space.eq_w(val, space.wrap(42)) -def test_execwith_withvars(space): +def test_execwith_withvars(space): val = space.appexec([space.wrap(7)], """ - (x): - y = 6 * x - return y - """) + (x): + y = 6 * x + return y + """) assert space.eq_w(val, space.wrap(42)) -def test_execwith_compile_error(space): +def test_execwith_compile_error(space): excinfo = py.test.raises(OperationError, space.appexec, [], """ - (): - y y + (): + y y """) # NOTE: the following test only works because excinfo.value is not # normalized so far - assert str(excinfo.value.get_w_value(space)).find('y y') != -1 + assert str(excinfo.value.get_w_value(space)).find('y y') != -1 def test_simple_applevel(space): - app = appdef("""app(x,y): + app = appdef("""app(x,y): return x + y """) assert app.func_name == 'app' @@ -36,15 +36,15 @@ assert space.eq_w(w_result, space.wrap(42)) def test_applevel_with_one_default(space): - app = appdef("""app(x,y=1): + app = appdef("""app(x,y=1): return x + y """) assert app.func_name == 'app' - w_result = app(space, space.wrap(41)) + w_result = app(space, space.wrap(41)) assert space.eq_w(w_result, space.wrap(42)) def test_applevel_with_two_defaults(space): - app = appdef("""app(x=1,y=2): + app = appdef("""app(x=1,y=2): return x + y """) w_result = app(space, space.wrap(41), space.wrap(1)) @@ -58,19 +58,19 @@ def test_applevel_noargs(space): - app = appdef("""app(): - return 42 + app = appdef("""app(): + return 42 """) assert app.func_name == 'app' - w_result = app(space) + w_result = app(space) assert space.eq_w(w_result, space.wrap(42)) -def somefunc(arg2=42): - return arg2 +def somefunc(arg2=42): + return arg2 -def test_app2interp_somefunc(space): - app = appdef(somefunc) - w_result = app(space) +def test_app2interp_somefunc(space): + app = appdef(somefunc) + w_result = app(space) assert space.eq_w(w_result, space.wrap(42)) def test_applevel_functions(space, applevel_temp = applevel_temp): @@ -87,45 +87,45 @@ def test_applevel_class(space, applevel_temp = applevel_temp): app = applevel_temp(''' class C(object): - clsattr = 42 - def __init__(self, x=13): - self.attr = x + clsattr = 42 + def __init__(self, x=13): + self.attr = x ''') C = app.interphook('C') - c = C(space, space.wrap(17)) + c = C(space, space.wrap(17)) w_attr = space.getattr(c, space.wrap('clsattr')) assert space.eq_w(w_attr, space.wrap(42)) w_clsattr = space.getattr(c, space.wrap('attr')) assert space.eq_w(w_clsattr, space.wrap(17)) -def app_test_something_at_app_level(): +def app_test_something_at_app_level(): x = 2 assert x/2 == 1 -class AppTestMethods: - def test_some_app_test_method(self): +class AppTestMethods: + def test_some_app_test_method(self): assert 2 == 2 -class TestMixedModule: - def test_accesses(self): +class TestMixedModule: + def test_accesses(self): space = self.space - import demomixedmod - w_module = demomixedmod.Module(space, space.wrap('mixedmodule')) + from .demomixedmod.moduledef import Module + w_module = Module(space, space.wrap('mixedmodule')) space.appexec([w_module], """ - (module): - assert module.value is None + (module): + assert module.value is None assert module.__doc__ == 'mixedmodule doc' - assert module.somefunc is module.somefunc - result = module.somefunc() - assert result == True + assert module.somefunc is module.somefunc + result = module.somefunc() + assert result == True - assert module.someappfunc is module.someappfunc - appresult = module.someappfunc(41) - assert appresult == 42 + assert module.someappfunc is module.someappfunc + appresult = module.someappfunc(41) + assert appresult == 42 assert module.__dict__ is module.__dict__ - for name in ('somefunc', 'someappfunc', '__doc__', '__name__'): + for name in ('somefunc', 'someappfunc', '__doc__', '__name__'): assert name in module.__dict__ """) assert space.is_true(w_module.call('somefunc')) diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py --- a/pypy/module/_demo/test/test_import.py +++ b/pypy/module/_demo/test/test_import.py @@ -1,30 +1,30 @@ -from pypy.module import _demo +from pypy.module._demo.moduledef import Module from pypy.tool.option import make_config, make_objspace class TestImport: def setup_method(self, func): - _demo.Module.demo_events = [] + Module.demo_events = [] def test_startup(self): config = make_config(None, usemodules=('_demo',)) space = make_objspace(config) w_modules = space.sys.get('modules') - assert _demo.Module.demo_events == ['setup'] + assert Module.demo_events == ['setup'] assert not space.contains_w(w_modules, space.wrap('_demo')) # first import w_import = space.builtin.get('__import__') w_demo = space.call(w_import, space.newlist([space.wrap('_demo')])) - assert _demo.Module.demo_events == ['setup', 'startup'] + assert Module.demo_events == ['setup', 'startup'] # reload the module, this should not call startup again space.delitem(w_modules, space.wrap('_demo')) w_demo = space.call(w_import, space.newlist([space.wrap('_demo')])) - assert _demo.Module.demo_events == ['setup', 'startup'] + assert Module.demo_events == ['setup', 'startup'] assert space.getattr(w_demo, space.wrap('measuretime')) From pypy.commits at gmail.com Sun Nov 11 00:01:00 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 10 Nov 2018 21:01:00 -0800 (PST) Subject: [pypy-commit] pypy default: fix test on win32, also cleanup uneeded export of function from libpypy Message-ID: <5be7b78c.1c69fb81.54898.ec58@mx.google.com> Author: Matti Picus Branch: Changeset: r95291:286138f59bc3 Date: 2018-11-10 20:55 -0800 http://bitbucket.org/pypy/pypy/changeset/286138f59bc3/ Log: fix test on win32, also cleanup uneeded export of function from libpypy diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) From pypy.commits at gmail.com Sun Nov 11 00:01:02 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 10 Nov 2018 21:01:02 -0800 (PST) Subject: [pypy-commit] pypy py3.5: silence print statements in tests Message-ID: <5be7b78e.1c69fb81.abbd5.80e4@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95292:f6d45acb0aaf Date: 2018-11-02 20:09 +0200 http://bitbucket.org/pypy/pypy/changeset/f6d45acb0aaf/ Log: silence print statements in tests diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -169,12 +169,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py --- a/pypy/module/_io/test/test_bytesio.py +++ b/pypy/module/_io/test/test_bytesio.py @@ -111,12 +111,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # b.close() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -164,12 +164,12 @@ # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, f.readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() From pypy.commits at gmail.com Sun Nov 11 04:05:16 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:16 -0800 (PST) Subject: [pypy-commit] pypy default: error out if vcvars.bat fails Message-ID: <5be7f0cc.1c69fb81.9fa27.c27b@mx.google.com> Author: Matti Picus Branch: Changeset: r95293:dbfbfcf077e9 Date: 2018-11-04 22:54 -0500 http://bitbucket.org/pypy/pypy/changeset/dbfbfcf077e9/ Log: error out if vcvars.bat fails diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -88,6 +88,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Sun Nov 11 04:05:18 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:18 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5be7f0ce.1c69fb81.855c7.3924@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95294:27c903e0884f Date: 2018-11-04 22:55 -0500 http://bitbucket.org/pypy/pypy/changeset/27c903e0884f/ Log: merge default into branch diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -85,6 +85,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Sun Nov 11 04:05:19 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:19 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5be7f0cf.1c69fb81.4977a.aabc@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r95295:4efd030655e5 Date: 2018-11-04 22:57 -0500 http://bitbucket.org/pypy/pypy/changeset/4efd030655e5/ Log: merge default into branch diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -88,6 +88,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Sun Nov 11 04:05:21 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:21 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge unicode-utf8 into branch Message-ID: <5be7f0d1.1c69fb81.86ec5.77ca@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95296:89183b3a84f3 Date: 2018-11-04 22:58 -0500 http://bitbucket.org/pypy/pypy/changeset/89183b3a84f3/ Log: merge unicode-utf8 into branch diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -85,6 +85,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Sun Nov 11 04:05:23 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:23 -0800 (PST) Subject: [pypy-commit] pypy default: fix tests to pass on win32 Message-ID: <5be7f0d3.1c69fb81.f8cdb.3098@mx.google.com> Author: Matti Picus Branch: Changeset: r95297:956dce5d3456 Date: 2018-11-11 00:25 -0800 http://bitbucket.org/pypy/pypy/changeset/956dce5d3456/ Log: fix tests to pass on win32 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -211,9 +211,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -576,6 +576,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): From pypy.commits at gmail.com Sun Nov 11 04:05:25 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:25 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge heads Message-ID: <5be7f0d5.1c69fb81.12717.f338@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95298:aea8f2fd3593 Date: 2018-11-11 00:50 -0800 http://bitbucket.org/pypy/pypy/changeset/aea8f2fd3593/ Log: merge heads diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -85,6 +85,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Sun Nov 11 04:05:26 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Nov 2018 01:05:26 -0800 (PST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5be7f0d6.1c69fb81.12717.f33c@mx.google.com> Author: Matti Picus Branch: Changeset: r95299:3e0eab4be623 Date: 2018-11-11 01:03 -0800 http://bitbucket.org/pypy/pypy/changeset/3e0eab4be623/ Log: merge heads diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -88,6 +88,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Mon Nov 12 11:56:08 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 12 Nov 2018 08:56:08 -0800 (PST) Subject: [pypy-commit] pypy default: #2912 Message-ID: <5be9b0a8.1c69fb81.50945.965e@mx.google.com> Author: Armin Rigo Branch: Changeset: r95300:01d5d6b5425e Date: 2018-11-12 17:55 +0100 http://bitbucket.org/pypy/pypy/changeset/01d5d6b5425e/ Log: #2912 Rewrite PyUnicode_FromEncodedObject() to use CPython's logic (still needs to be forward-ported to py3.5) diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -421,6 +421,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -366,10 +366,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -398,28 +402,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): From pypy.commits at gmail.com Mon Nov 12 16:18:13 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 12 Nov 2018 13:18:13 -0800 (PST) Subject: [pypy-commit] pypy py3tests: hg merge apptest-file Message-ID: <5be9ee15.1c69fb81.ccab6.492c@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r95301:399533825dcd Date: 2018-11-12 21:17 +0000 http://bitbucket.org/pypy/pypy/changeset/399533825dcd/ Log: hg merge apptest-file diff --git a/pypy/interpreter/test/demomixedmod/__init__.py b/pypy/interpreter/test/demomixedmod/__init__.py --- a/pypy/interpreter/test/demomixedmod/__init__.py +++ b/pypy/interpreter/test/demomixedmod/__init__.py @@ -1,15 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - interpleveldefs = { - '__name__' : '(space.wrap("mixedmodule"))', - '__doc__' : '(space.wrap("mixedmodule doc"))', - 'somefunc' : 'file1.somefunc', - 'value' : '(space.w_None)', - 'path' : 'file1.initpath(space)', - 'cpypath' : 'space.wrap(sys.path)' - } - - appleveldefs = { - 'someappfunc' : 'file2_app.someappfunc', - } diff --git a/pypy/interpreter/test/demomixedmod/__init__.py b/pypy/interpreter/test/demomixedmod/moduledef.py copy from pypy/interpreter/test/demomixedmod/__init__.py copy to pypy/interpreter/test/demomixedmod/moduledef.py diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py --- a/pypy/interpreter/test/test_appinterp.py +++ b/pypy/interpreter/test/test_appinterp.py @@ -3,32 +3,32 @@ from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp from pypy.interpreter.error import OperationError -def test_execwith_novars(space): - val = space.appexec([], """ - (): - return 42 - """) +def test_execwith_novars(space): + val = space.appexec([], """ + (): + return 42 + """) assert space.eq_w(val, space.wrap(42)) -def test_execwith_withvars(space): +def test_execwith_withvars(space): val = space.appexec([space.wrap(7)], """ - (x): - y = 6 * x - return y - """) + (x): + y = 6 * x + return y + """) assert space.eq_w(val, space.wrap(42)) -def test_execwith_compile_error(space): +def test_execwith_compile_error(space): excinfo = py.test.raises(OperationError, space.appexec, [], """ - (): - y y + (): + y y """) # NOTE: the following test only works because excinfo.value is not # normalized so far - assert str(excinfo.value.get_w_value(space)).find('y y') != -1 + assert str(excinfo.value.get_w_value(space)).find('y y') != -1 def test_simple_applevel(space): - app = appdef("""app(x,y): + app = appdef("""app(x,y): return x + y """) assert app.__name__ == 'app' @@ -36,15 +36,15 @@ assert space.eq_w(w_result, space.wrap(42)) def test_applevel_with_one_default(space): - app = appdef("""app(x,y=1): + app = appdef("""app(x,y=1): return x + y """) assert app.__name__ == 'app' - w_result = app(space, space.wrap(41)) + w_result = app(space, space.wrap(41)) assert space.eq_w(w_result, space.wrap(42)) def test_applevel_with_two_defaults(space): - app = appdef("""app(x=1,y=2): + app = appdef("""app(x=1,y=2): return x + y """) w_result = app(space, space.wrap(41), space.wrap(1)) @@ -58,19 +58,19 @@ def test_applevel_noargs(space): - app = appdef("""app(): - return 42 + app = appdef("""app(): + return 42 """) assert app.__name__ == 'app' - w_result = app(space) + w_result = app(space) assert space.eq_w(w_result, space.wrap(42)) -def somefunc(arg2=42): - return arg2 +def somefunc(arg2=42): + return arg2 -def test_app2interp_somefunc(space): - app = appdef(somefunc) - w_result = app(space) +def test_app2interp_somefunc(space): + app = appdef(somefunc) + w_result = app(space) assert space.eq_w(w_result, space.wrap(42)) def test_applevel_functions(space, applevel_temp = applevel_temp): @@ -87,45 +87,45 @@ def test_applevel_class(space, applevel_temp = applevel_temp): app = applevel_temp(''' class C(object): - clsattr = 42 - def __init__(self, x=13): - self.attr = x + clsattr = 42 + def __init__(self, x=13): + self.attr = x ''') C = app.interphook('C') - c = C(space, space.wrap(17)) + c = C(space, space.wrap(17)) w_attr = space.getattr(c, space.wrap('clsattr')) assert space.eq_w(w_attr, space.wrap(42)) w_clsattr = space.getattr(c, space.wrap('attr')) assert space.eq_w(w_clsattr, space.wrap(17)) -def app_test_something_at_app_level(): +def app_test_something_at_app_level(): x = 2 assert x/2 == 1 -class AppTestMethods: - def test_some_app_test_method(self): +class AppTestMethods: + def test_some_app_test_method(self): assert 2 == 2 -class TestMixedModule: - def test_accesses(self): +class TestMixedModule: + def test_accesses(self): space = self.space - import demomixedmod - w_module = demomixedmod.Module(space, space.wrap('mixedmodule')) + from .demomixedmod.moduledef import Module + w_module = Module(space, space.wrap('mixedmodule')) space.appexec([w_module], """ - (module): - assert module.value is None + (module): + assert module.value is None assert module.__doc__ == 'mixedmodule doc' - assert module.somefunc is module.somefunc - result = module.somefunc() - assert result == True + assert module.somefunc is module.somefunc + result = module.somefunc() + assert result == True - assert module.someappfunc is module.someappfunc - appresult = module.someappfunc(41) - assert appresult == 42 + assert module.someappfunc is module.someappfunc + appresult = module.someappfunc(41) + assert appresult == 42 assert module.__dict__ is module.__dict__ - for name in ('somefunc', 'someappfunc', '__doc__', '__name__'): + for name in ('somefunc', 'someappfunc', '__doc__', '__name__'): assert name in module.__dict__ """) assert space.is_true(w_module.call('somefunc')) diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -31,7 +31,8 @@ cif_descr = lltype.nullptr(CIF_DESCRIPTION) - def __init__(self, space, fargs, fresult, ellipsis, abi=FFI_DEFAULT_ABI): + def __init__(self, space, fargs, fresult, ellipsis, + abi=FFI_DEFAULT_ABI): assert isinstance(ellipsis, bool) extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi) size = rffi.sizeof(rffi.VOIDP) @@ -98,7 +99,6 @@ lltype.free(self.cif_descr, flavor='raw') def _compute_extra_text(self, fargs, fresult, ellipsis, abi): - from pypy.module._cffi_backend import newtype argnames = ['(*)('] xpos = 2 if has_stdcall and abi == FFI_STDCALL: diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -6,8 +6,7 @@ from rpython.rlib import jit, rgc from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.module._cffi_backend.moduledef import ( - get_dict_rtld_constants) +from pypy.module._cffi_backend.moduledef import get_dict_rtld_constants from pypy.module._cffi_backend import parse_c_type, realize_c_type from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle diff --git a/pypy/module/_cffi_backend/moduledef.py b/pypy/module/_cffi_backend/moduledef.py --- a/pypy/module/_cffi_backend/moduledef.py +++ b/pypy/module/_cffi_backend/moduledef.py @@ -2,7 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -from pypy.module._cffi_backend import VERSION + +from . import VERSION FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -686,9 +686,9 @@ # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError, # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError, # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ... - from pypy.module.exceptions.moduledef import Module global all_exceptions - all_exceptions = list(Module.interpleveldefs) + from pypy.module.exceptions.moduledef import Module as ExcModule + all_exceptions = list(ExcModule.interpleveldefs) for exc_name in all_exceptions: if exc_name in ('EnvironmentError', 'IOError', 'WindowsError'): # FIXME: aliases of OSError cause a clash of names via diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -7,7 +7,6 @@ from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, from_ref, incref, decref, get_typedescr, hack_for_result_often_existing_obj) -from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.bytesobject import invoke_bytes_method @@ -267,7 +266,7 @@ if opid == Py_EQ: return 1 if opid == Py_NE: - return 0 + return 0 w_res = PyObject_RichCompare(space, w_o1, w_o2, opid_int) return int(space.is_true(w_res)) diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -14,6 +14,8 @@ from rpython.rlib import rawrefcount from rpython.tool.udir import udir +import pypy.module.cpyext.moduledef # Make sure all the functions are registered + only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) From pypy.commits at gmail.com Mon Nov 12 16:46:55 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Nov 2018 13:46:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5be9f4cf.1c69fb81.54898.b495@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95302:2a37ce3c7d2f Date: 2018-11-12 09:22 -0800 http://bitbucket.org/pypy/pypy/changeset/2a37ce3c7d2f/ Log: merge default into py3.5 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -684,6 +684,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -486,10 +486,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newtext(rffi.charp2str(errors)) @@ -525,28 +529,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -227,9 +227,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print(type(st).__module__) + # print(type(st).__module__) s = pickle.dumps(st) - print(repr(s)) + # print(repr(s)) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -573,6 +573,12 @@ res = fp.read() assert res == '1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): # nt specific diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -236,8 +236,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -273,7 +273,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -291,11 +290,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) From pypy.commits at gmail.com Mon Nov 12 16:46:57 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 12 Nov 2018 13:46:57 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix merge, PyUnicode_FromEncodedObject already tested in test_decode Message-ID: <5be9f4d1.1c69fb81.58fab.32a3@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95303:861c793cead8 Date: 2018-11-12 13:39 -0800 http://bitbucket.org/pypy/pypy/changeset/861c793cead8/ Log: fix merge, PyUnicode_FromEncodedObject already tested in test_decode diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -684,8 +684,6 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) - assert space.unicode_w(PyUnicode_FromEncodedObject( - space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -490,11 +490,12 @@ encoding, errors) def _pyunicode_decode(space, s, encoding, errors): - if not encoding: - # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead - encoding = PyUnicode_GetDefaultEncoding(space) + if encoding: + w_encoding = space.newtext(rffi.charp2str(encoding)) + else: + # python 3.4 changed to this from defaultencoding + w_encoding = space.newtext('utf-8') w_str = space.newbytes(s) - w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newtext(rffi.charp2str(errors)) else: @@ -530,10 +531,10 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" if space.isinstance_w(w_obj, space.w_unicode): - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") + raise oefmt(space.w_TypeError, "decoding str is not supported") if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") - s = space.bufferstr_w(w_obj) + s = space.bytes_w(w_obj) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Tue Nov 13 03:19:51 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:19:51 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bea8927.1c69fb81.72596.6371@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95304:e41d802a74a7 Date: 2018-11-12 16:23 -0800 http://bitbucket.org/pypy/pypy/changeset/e41d802a74a7/ Log: merge py3.5 into branch diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -169,12 +169,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py --- a/pypy/module/_io/test/test_bytesio.py +++ b/pypy/module/_io/test/test_bytesio.py @@ -111,12 +111,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # b.close() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -164,12 +164,12 @@ # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, f.readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -495,12 +495,16 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" - if not encoding: - # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): + if encoding: + w_encoding = space.newtext(rffi.charp2str(encoding)) + else: + # python 3.4 changed to this from defaultencoding w_encoding = space.newtext('utf-8') - else: - w_encoding = space.newtext(rffi.charp2str(encoding)) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) if errors: w_errors = space.newtext(rffi.charp2str(errors)) else: @@ -535,28 +539,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + raise oefmt(space.w_TypeError, "decoding str is not supported") + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bytes_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -226,9 +226,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print(type(st).__module__) + # print(type(st).__module__) s = pickle.dumps(st) - print(repr(s)) + # print(repr(s)) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -572,6 +572,12 @@ res = fp.read() assert res == '1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): # nt specific diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -236,8 +236,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -273,7 +273,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -291,11 +290,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) From pypy.commits at gmail.com Tue Nov 13 03:19:53 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:19:53 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix to skip test Message-ID: <5bea8929.1c69fb81.e081c.cdb5@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95305:debe241b1b92 Date: 2018-11-12 19:20 -0800 http://bitbucket.org/pypy/pypy/changeset/debe241b1b92/ Log: fix to skip test diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -532,9 +532,9 @@ w_u = space.wrap(u'späm') try: w_s = PyUnicode_EncodeFSDefault(space, w_u) - except OperationError: + except (OperationError, UnicodeEncodeError): py.test.skip("Requires a unicode-aware fsencoding") - with rffi.scoped_str2charp(space.text_w(w_s)) as encoded: + with rffi.scoped_str2charp(space.bytes_w(w_s)) as encoded: w_decoded = PyUnicode_DecodeFSDefaultAndSize(space, encoded, space.len_w(w_s)) assert space.eq_w(w_decoded, w_u) w_decoded = PyUnicode_DecodeFSDefault(space, encoded) From pypy.commits at gmail.com Tue Nov 13 03:19:55 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:19:55 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: use utf8 without converting to unicode Message-ID: <5bea892b.1c69fb81.7d329.8814@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95306:5e342e9eee98 Date: 2018-11-12 19:23 -0800 http://bitbucket.org/pypy/pypy/changeset/5e342e9eee98/ Log: use utf8 without converting to unicode diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -460,14 +460,14 @@ def surrogateescape_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + utf8 = space.utf8_w(space.getattr(w_exc, space.newtext('object'))) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) res = '' pos = start while pos < end: - ch = ord(obj[pos]) + ch = rutf8.codepoint_at_pos(utf8, pos) pos += 1 if ch < 0xdc80 or ch > 0xdcff: # Not a UTF-8b surrogate, fail with original exception From pypy.commits at gmail.com Tue Nov 13 03:19:56 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:19:56 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: special case surrogateescape if already handled, this code needs refactoring Message-ID: <5bea892c.1c69fb81.da359.5e1a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95307:88ab1ff0c4e8 Date: 2018-11-12 19:25 -0800 http://bitbucket.org/pypy/pypy/changeset/88ab1ff0c4e8/ Log: special case surrogateescape if already handled, this code needs refactoring diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1206,10 +1206,13 @@ from pypy.module._codecs.interp_codecs import encode_text, CodecState utf8 = space.utf8_w(w_object) if not allow_surrogates: - utf8 = space.utf8_w(w_object) if errors is None: errors = 'strict' pos = rutf8.surrogate_in_utf8(utf8) + if pos >= 0: + handled_error = True + else: + handled_error = False state = space.fromcache(CodecState) eh = state.encode_error_handler while pos >= 0: @@ -1224,7 +1227,7 @@ # surrogatepass? break pos = _pos - if errors == 'surrogateescape': + if errors == 'surrogateescape' and handled_error: #escape return space.newbytes(utf8) w_object = space.newtext(utf8) From pypy.commits at gmail.com Tue Nov 13 03:19:58 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:19:58 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix for surrogates in type name Message-ID: <5bea892e.1c69fb81.40507.7394@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95308:720cb17cdc85 Date: 2018-11-12 20:29 -0800 http://bitbucket.org/pypy/pypy/changeset/720cb17cdc85/ Log: test, fix for surrogates in type name diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -942,12 +942,15 @@ assert Abc.__name__ == 'Def' raises(TypeError, "Abc.__name__ = 42") raises(TypeError, "Abc.__name__ = b'A'") - try: - Abc.__name__ = 'G\x00hi' - except ValueError as e: - assert str(e) == "type name must not contain null characters" - else: - assert False + for v, err in [('G\x00hi', "type name must not contain null characters"), + ('A\udcdcB', "surrogates not allowed")]: + try: + Abc.__name__ = v + except ValueError as e: + assert err in str(e) + else: + assert False + assert Abc.__name__ == 'Def' def test_qualname(self): A = type('A', (), {'__qualname__': 'B.C'}) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -14,7 +14,7 @@ from rpython.rlib.objectmodel import current_object_addr_as_int, compute_hash from rpython.rlib.objectmodel import we_are_translated, not_rpython from rpython.rlib.rarithmetic import intmask, r_uint -from rpython.rlib.rutf8 import CheckError, check_utf8 +from rpython.rlib.rutf8 import CheckError, check_utf8, surrogate_in_utf8 class MutableCell(W_Root): def unwrap_cell(self, space): @@ -801,9 +801,12 @@ return space.call_function(newfunc, w_winner, w_name, w_bases, w_dict) w_typetype = w_winner - name = space.text_w(w_name) # NB. CPython forbids surrogates here + name = space.text_w(w_name) if '\x00' in name: raise oefmt(space.w_ValueError, "type name must not contain null characters") + if surrogate_in_utf8(name) >= 0: + raise oefmt(space.w_ValueError, "can't encode character %c in position " + "%i, surrogates not allowed", name[pos], pos) dict_w = {} dictkeys_w = space.listview(w_dict) for w_key in dictkeys_w: @@ -868,6 +871,10 @@ name = space.text_w(w_value) if '\x00' in name: raise oefmt(space.w_ValueError, "type name must not contain null characters") + pos = surrogate_in_utf8(name) + if pos >= 0: + raise oefmt(space.w_ValueError, "can't encode character %s in position " + "%d, surrogates not allowed", name[pos], pos) w_type.name = name def descr_get__qualname__(space, w_type): From pypy.commits at gmail.com Tue Nov 13 03:20:00 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:20:00 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix assigning to type qualname Message-ID: <5bea8930.1c69fb81.6ec71.89df@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95309:9b6e8faed6ae Date: 2018-11-12 20:36 -0800 http://bitbucket.org/pypy/pypy/changeset/9b6e8faed6ae/ Log: test, fix assigning to type qualname diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -971,6 +971,14 @@ e = raises(TypeError, type, 'D', (), {'__qualname__': 42}) assert str(e.value) == "type __qualname__ must be a str, not int" + for v in (42, b'abc'): + try: + C.__qualname__ = v + except TypeError as e: + assert 'can only assign string' in str(e) + else: + assert False + def test_compare(self): class A(object): pass diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -885,6 +885,10 @@ w_type = _check(space, w_type) if not w_type.is_heaptype(): raise oefmt(space.w_TypeError, "can't set %N.__qualname__", w_type) + if not space.isinstance_w(w_value, space.w_text): + raise oefmt(space.w_TypeError, + "can only assign string to %N.__name__, not '%T'", + w_type, w_value) w_type.qualname = space.utf8_w(w_value) def descr_get__mro__(space, w_type): From pypy.commits at gmail.com Tue Nov 13 03:20:02 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:20:02 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix wcharp2utf8n for lastchr != 0 Message-ID: <5bea8932.1c69fb81.31fb2.ca7d@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95310:89c980f7f54b Date: 2018-11-13 00:18 -0800 http://bitbucket.org/pypy/pypy/changeset/89c980f7f54b/ Log: test, fix wcharp2utf8n for lastchr != 0 diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -1046,7 +1046,7 @@ s = rutf8.Utf8StringBuilder(maxlen) i = 0 - while i < maxlen and w[i]: + while i < maxlen and w[i] != '\x00': s.append_code(ord(w[i])) i += 1 return s.build(), i diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -916,3 +916,8 @@ assert buf[1] == 'a' assert buf[2] == 'r' assert buf[3] == '\x00' + +def test_wcharp2utf8n(): + w = 'hello\x00\x00\x00\x00' + u, i = wcharp2utf8n(w, len(w)) + assert i == len('hello') From pypy.commits at gmail.com Tue Nov 13 03:52:38 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 00:52:38 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix copy-paste Message-ID: <5bea90d6.1c69fb81.cea5a.d211@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95311:3eafff66d59a Date: 2018-11-13 00:51 -0800 http://bitbucket.org/pypy/pypy/changeset/3eafff66d59a/ Log: fix copy-paste diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -804,7 +804,8 @@ name = space.text_w(w_name) if '\x00' in name: raise oefmt(space.w_ValueError, "type name must not contain null characters") - if surrogate_in_utf8(name) >= 0: + pos = surrogate_in_utf8(name) + if pos >= 0: raise oefmt(space.w_ValueError, "can't encode character %c in position " "%i, surrogates not allowed", name[pos], pos) dict_w = {} From pypy.commits at gmail.com Tue Nov 13 06:09:05 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 13 Nov 2018 03:09:05 -0800 (PST) Subject: [pypy-commit] pypy default: add_memory_pressure() tweaks Message-ID: <5beab0d1.1c69fb81.e5fc1.3cea@mx.google.com> Author: Armin Rigo Branch: Changeset: r95312:d7968c7867d0 Date: 2018-11-13 13:08 +0200 http://bitbucket.org/pypy/pypy/changeset/d7968c7867d0/ Log: add_memory_pressure() tweaks diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), From pypy.commits at gmail.com Tue Nov 13 09:11:41 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 06:11:41 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix translation Message-ID: <5beadb9d.1c69fb81.eaf2.e0ab@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95313:f8e25026d0af Date: 2018-11-13 06:08 -0800 http://bitbucket.org/pypy/pypy/changeset/f8e25026d0af/ Log: fix translation diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -806,8 +806,8 @@ raise oefmt(space.w_ValueError, "type name must not contain null characters") pos = surrogate_in_utf8(name) if pos >= 0: - raise oefmt(space.w_ValueError, "can't encode character %c in position " - "%i, surrogates not allowed", name[pos], pos) + raise oefmt(space.w_ValueError, "can't encode character %s in position " + "%d, surrogates not allowed", name[pos], pos) dict_w = {} dictkeys_w = space.listview(w_dict) for w_key in dictkeys_w: From pypy.commits at gmail.com Tue Nov 13 09:11:44 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 13 Nov 2018 06:11:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5beadba0.1c69fb81.a8bf1.fe58@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95314:3bb86a88e1d3 Date: 2018-11-13 06:10 -0800 http://bitbucket.org/pypy/pypy/changeset/3bb86a88e1d3/ Log: merge default into branch diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -179,6 +179,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -191,14 +195,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -215,6 +218,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -259,8 +267,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), From pypy.commits at gmail.com Wed Nov 14 14:09:10 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 14 Nov 2018 11:09:10 -0800 (PST) Subject: [pypy-commit] pypy apptest-file: Avoid importing rpython when running -A tests Message-ID: <5bec72d6.1c69fb81.487ad.ff8d@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r95315:4f36bf5a1d30 Date: 2018-11-14 19:08 +0000 http://bitbucket.org/pypy/pypy/changeset/4f36bf5a1d30/ Log: Avoid importing rpython when running -A tests diff --git a/pypy/module/_continuation/test/conftest.py b/pypy/module/_continuation/test/conftest.py --- a/pypy/module/_continuation/test/conftest.py +++ b/pypy/module/_continuation/test/conftest.py @@ -1,7 +1,7 @@ -import pytest import sys def pytest_configure(config): - if sys.platform.startswith('linux'): + if (not config.getoption('runappdirect') and + sys.platform.startswith('linux')): from rpython.rlib.rvmprof.cintf import configure_libbacktrace_linux configure_libbacktrace_linux() diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -37,6 +37,8 @@ disabled = None def pytest_configure(config): + if config.getoption('runappdirect'): + return if py.path.local.sysfind('genreflex') is None: import pypy.module._cppyy.capi.loadable_capi as lcapi try: diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -16,10 +16,9 @@ space = gettestobjspace(usemodules=['time']) space.getbuiltinmodule("time") -def pytest_ignore_collect(path, config): # ensure additional functions are registered import pypy.module.cpyext.test.test_cpyext - return False + def pytest_funcarg__api(request): return request.cls.api diff --git a/pypy/module/pypyjit/test/conftest.py b/pypy/module/pypyjit/test/conftest.py deleted file mode 100644 --- a/pypy/module/pypyjit/test/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -def pytest_addoption(parser): - group = parser.getgroup("pypyjit options") - group.addoption("--pypy", action="store", default=None, dest="pypy_c", - help="DEPRECATED: use this in test_pypy_c instead") -# XXX kill the corresponding section in the buildbot run, -# which (as far as I can tell) ignores that option entirely and does -# the same as the regular py.test. From pypy.commits at gmail.com Thu Nov 15 04:12:04 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 15 Nov 2018 01:12:04 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: pos in encoding error handler is in unicode not utf8. refactor, use only utf8 Message-ID: <5bed3864.1c69fb81.deabb.6030@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95318:83e8a364e912 Date: 2018-11-15 01:11 -0800 http://bitbucket.org/pypy/pypy/changeset/83e8a364e912/ Log: pos in encoding error handler is in unicode not utf8. refactor, use only utf8 diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -241,8 +241,9 @@ pos = e.pos assert pos >= 0 start = s[:pos] + upos = rutf8.codepoints_in_utf8(s, end=pos) ru, lgt = errorhandler(errors, 'utf8', - 'surrogates not allowed', s, pos, pos + 1) + 'surrogates not allowed', s, upos, upos + 1) end = utf8_encode_utf_8(s[pos+3:], errors, errorhandler, allow_surrogates=allow_surrogates) s = start + ru + end diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -379,10 +379,11 @@ def surrogatepass_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + utf8 = space.utf8_w(space.getattr(w_exc, space.newtext('object'))) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) encoding = space.text_w(space.getattr(w_exc, space.newtext('encoding'))) + msg = space.text_w(space.getattr(w_exc, space.newtext('reason'))) bytelength, code = get_standard_encoding(encoding) if code == ENC_UNKNOWN: # Not supported, fail with original exception @@ -390,8 +391,12 @@ end = space.int_w(w_end) builder = StringBuilder() pos = start + # start, end are in codepoint indices + itr = rutf8.Utf8StringIterator(utf8) + for i in range(pos): + itr.next() while pos < end: - ch = ord(obj[pos]) + ch = itr.next() pos += 1 if ch < 0xd800 or ch > 0xdfff: # Not a surrogate, fail with original exception @@ -466,8 +471,11 @@ end = space.int_w(w_end) res = '' pos = start + itr = rutf8.Utf8StringIterator(utf8) + for i in range(pos): + itr.next() while pos < end: - ch = rutf8.codepoint_at_pos(utf8, pos) + ch = itr.next() pos += 1 if ch < 0xdc80 or ch > 0xdcff: # Not a UTF-8b surrogate, fail with original exception diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -648,6 +648,8 @@ assert u'\ud8ae'.encode('utf_16_be', 'surrogatepass') == b'\xd8\xae' assert (u'\U0000d8ae'.encode('utf-32-be', 'surrogatepass') == b'\x00\x00\xd8\xae') + assert (u'\x80\ud800'.encode('utf8', 'surrogatepass') == + b'\xc2\x80\xed\xa0\x80') def test_badandgoodsurrogatepassexceptions(self): import codecs diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1209,28 +1209,26 @@ if errors is None: errors = 'strict' pos = rutf8.surrogate_in_utf8(utf8) - if pos >= 0: - handled_error = True - else: - handled_error = False state = space.fromcache(CodecState) eh = state.encode_error_handler - while pos >= 0: - start = utf8[:pos] - ru, _pos = eh(errors, "utf8", "surrogates not allowed", utf8, - pos, pos + 1) - upos = rutf8.next_codepoint_pos(utf8, _pos) - end = utf8[upos:] - utf8 = start + ru + end - _pos = rutf8.surrogate_in_utf8(utf8) - if _pos <= pos: - # surrogatepass? - break - pos = _pos - if errors == 'surrogateescape' and handled_error: - #escape - return space.newbytes(utf8) - w_object = space.newtext(utf8) + if pos >= 0: + while pos >= 0: + start = utf8[:pos] + upos = rutf8.codepoints_in_utf8(utf8, end=pos) + ru, _pos = eh(errors, "utf8", "surrogates not allowed", utf8, + upos, upos + 1) + upos = rutf8.next_codepoint_pos(utf8, _pos) + end = utf8[upos:] + utf8 = start + ru + end + _pos = rutf8.surrogate_in_utf8(utf8) + if _pos <= pos: + # surrogatepass? + break + pos = _pos + if errors == 'surrogateescape': + #escape + return space.newbytes(utf8) + w_object = space.newtext(utf8) if errors is None or errors == 'strict': if encoding is None or encoding == 'utf-8': #if rutf8.has_surrogates(utf8): From pypy.commits at gmail.com Thu Nov 15 04:13:02 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 15 Nov 2018 01:13:02 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bed389e.1c69fb81.10c42.b7dd@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95319:5a159ecdb8a9 Date: 2018-11-15 01:12 -0800 http://bitbucket.org/pypy/pypy/changeset/5a159ecdb8a9/ Log: merge py3.5 into branch diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -179,6 +179,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -191,14 +195,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -215,6 +218,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -259,8 +267,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -689,6 +689,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.newbytes(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -543,7 +543,7 @@ raise oefmt(space.w_TypeError, "decoding str is not supported") if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") - s = space.bytes_w(w_obj) + s = space.charbuf_w(w_obj) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Thu Nov 15 13:11:56 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 15 Nov 2018 10:11:56 -0800 (PST) Subject: [pypy-commit] pypy apptest-file: Skip _cffi_backend C tests when versions don't match, instead of reporting umpteen failures Message-ID: <5bedb6ec.1c69fb81.8f2d1.c67d@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r95320:92562abdaf9f Date: 2018-11-15 18:11 +0000 http://bitbucket.org/pypy/pypy/changeset/92562abdaf9f/ Log: Skip _cffi_backend C tests when versions don't match, instead of reporting umpteen failures diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -16,8 +16,6 @@ 5. make the test pass in pypy ('py.test test_c.py') """ import py, sys, ctypes -if sys.version_info < (2, 6): - py.test.skip("requires the b'' literal syntax") from rpython.tool.udir import udir from pypy.interpreter import gateway @@ -27,6 +25,8 @@ from rpython.translator.platform import host from rpython.translator.tool.cbuild import ExternalCompilationInfo +from .. import VERSION as TEST_VERSION + class AppTestC(object): """Populated below, hack hack hack.""" @@ -34,6 +34,15 @@ spaceconfig = dict(usemodules=('_cffi_backend', 'cStringIO', 'array')) def setup_class(cls): + if cls.runappdirect: + _cffi_backend = py.test.importorskip('_cffi_backend') + if _cffi_backend.__version__ != TEST_VERSION: + py.test.skip( + "These tests are for cffi version %s, this Python " + "has version %s installed" % + (TEST_VERSION, _cffi_backend.__version__)) + + testfuncs_w = [] keepalive_funcs = [] UniqueCache.for_testing = True From pypy.commits at gmail.com Fri Nov 16 11:20:42 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 08:20:42 -0800 (PST) Subject: [pypy-commit] pypy default: Move -A only test to extra_tests/ Message-ID: <5beeee5a.1c69fb81.b36f8.11e3@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95321:3bb601076be2 Date: 2018-11-16 16:15 +0000 http://bitbucket.org/pypy/pypy/changeset/3bb601076be2/ Log: Move -A only test to extra_tests/ diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): From pypy.commits at gmail.com Fri Nov 16 11:33:11 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 08:33:11 -0800 (PST) Subject: [pypy-commit] pypy default: Workaround for a bug in pytest<3.0.5: https://github.com/pytest-dev/pytest/issues/2016 Message-ID: <5beef147.1c69fb81.f3abd.d0c7@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95322:51db06536f54 Date: 2018-11-16 16:32 +0000 http://bitbucket.org/pypy/pypy/changeset/51db06536f54/ Log: Workaround for a bug in pytest<3.0.5: https://github.com/pytest- dev/pytest/issues/2016 diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None From pypy.commits at gmail.com Fri Nov 16 11:54:44 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 08:54:44 -0800 (PST) Subject: [pypy-commit] pypy default: Don't mess up the warnings module for subsequent tests Message-ID: <5beef654.1c69fb81.8711c.03a5@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95323:ce46c3cdcf72 Date: 2018-11-16 16:53 +0000 http://bitbucket.org/pypy/pypy/changeset/ce46c3cdcf72/ Log: Don't mess up the warnings module for subsequent tests diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings From pypy.commits at gmail.com Fri Nov 16 14:56:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 11:56:40 -0800 (PST) Subject: [pypy-commit] pypy default: hg mv pypy/module/test_lib_pypy/cffi_tests/ extra_tests/ Message-ID: <5bef20f8.1c69fb81.198e4.cac5@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95325:d6d47a621c3c Date: 2018-11-16 19:11 +0000 http://bitbucket.org/pypy/pypy/changeset/d6d47a621c3c/ Log: hg mv pypy/module/test_lib_pypy/cffi_tests/ extra_tests/ diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py From pypy.commits at gmail.com Fri Nov 16 14:56:42 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 11:56:42 -0800 (PST) Subject: [pypy-commit] pypy default: Update import_cffi.py script and re-run it Message-ID: <5bef20fa.1c69fb81.2d2c6.4fa5@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95326:04bda091f27f Date: 2018-11-16 19:55 +0000 http://bitbucket.org/pypy/pypy/changeset/04bda091f27f/ Log: Update import_cffi.py script and re-run it diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/extra_tests/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py --- a/extra_tests/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/extra_tests/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/extra_tests/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py --- a/extra_tests/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/extra_tests/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py --- a/extra_tests/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/extra_tests/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/extra_tests/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/extra_tests/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/extra_tests/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/extra_tests/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/extra_tests/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/extra_tests/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py --- a/extra_tests/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py --- a/extra_tests/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py --- a/extra_tests/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/extra_tests/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py --- a/extra_tests/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/extra_tests/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py --- a/extra_tests/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/extra_tests/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py --- a/extra_tests/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/extra_tests/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py --- a/extra_tests/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/extra_tests/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py --- a/extra_tests/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/extra_tests/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py --- a/extra_tests/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/extra_tests/cffi_tests/support.py b/extra_tests/cffi_tests/support.py --- a/extra_tests/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): From pypy.commits at gmail.com Fri Nov 16 15:02:56 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 16 Nov 2018 12:02:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5bef2270.1c69fb81.4c39.9d37@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95327:10b8fc6af4a3 Date: 2018-11-16 20:02 +0000 http://bitbucket.org/pypy/pypy/changeset/10b8fc6af4a3/ Log: hg merge default diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1073,6 +1073,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -42,7 +42,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -75,7 +75,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -138,11 +138,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -156,7 +156,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print("Called", self.num) -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print(sys.executable, self.tmpfile) - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -61,18 +61,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = io.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = io.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): From pypy.commits at gmail.com Sat Nov 17 15:07:50 2018 From: pypy.commits at gmail.com (rlamy) Date: Sat, 17 Nov 2018 12:07:50 -0800 (PST) Subject: [pypy-commit] pypy default: Simplify tests Message-ID: <5bf07516.1c69fb81.4977a.1030@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95328:6891f3d45a36 Date: 2018-11-17 20:07 +0000 http://bitbucket.org/pypy/pypy/changeset/6891f3d45a36/ Log: Simplify tests diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - From pypy.commits at gmail.com Sun Nov 18 03:57:39 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 00:57:39 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 into branch Message-ID: <5bf12983.1c69fb81.87e3.c2a1@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95329:18106031c176 Date: 2018-11-17 23:33 -0800 http://bitbucket.org/pypy/pypy/changeset/18106031c176/ Log: merge py3.5 into branch diff too long, truncating to 2000 out of 2161 lines diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -8,11 +8,9 @@ available. """ -__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" - import sys import os -import imp +import imp, _imp from distutils.errors import DistutilsPlatformError @@ -71,9 +69,17 @@ def _init_nt(): """Initialize the module as appropriate for NT""" g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] # xxx? + g['VERSION'] = get_python_version().replace(".", "") + g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) global _config_vars _config_vars = g diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -190,7 +190,7 @@ # Prefer poll, if available, since you can poll() any fd # which can't be done with select(). if HAVE_POLL: - p.register(sock.fileno(), POLLOUT | POLLIN) + p.register(sock.fileno(), POLLOUT if writing else POLLIN) rc = len(p.poll(timeout * 1000.0)) else: diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -27,6 +27,12 @@ from .console import Console, Event from .unix_eventqueue import EventQueue from .trace import trace +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -76,8 +82,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -407,6 +413,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError) as err: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,16 @@ .. branch: py3.6-wordcode implement new wordcode instruction encoding on the 3.6 branch + +.. branch: socket_default_timeout_blockingness + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -110,6 +110,7 @@ 'stack_almost_full' : 'interp_magic.stack_almost_full', 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } submodules = { diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -209,3 +209,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -79,13 +79,6 @@ w_result = self.ctype.cast_to_int(ptr) return w_result - def long(self, space): - w_result = self.int(space) - space = self.space - if space.is_w(space.type(w_result), space.w_int): - w_result = space.newlong(space.int_w(w_result)) - return w_result - def float(self): with self as ptr: w_result = self.ctype.float(ptr) @@ -664,7 +657,6 @@ __repr__ = interp2app(W_CData.repr), __bool__ = interp2app(W_CData.bool), __int__ = interp2app(W_CData.int), - __long__ = interp2app(W_CData.long), __float__ = interp2app(W_CData.float), __complex__ = interp2app(W_CData.complex), __len__ = interp2app(W_CData.len), diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,24 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError +from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -202,6 +212,9 @@ else: if strict and value < 0: raise OperationError(space.w_OverflowError, space.newtext(neg_msg)) + if not we_are_translated(): + if isinstance(value, NonConstant): # hack for test_ztranslation + return r_uint(0) return r_uint(value) # note that if not 'strict', then space.int() will round down floats bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) @@ -389,3 +402,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,16 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend, sys + sys.pypy_initfsencoding() # initialize space.sys.filesystemencoding + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -761,6 +761,26 @@ assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -197,12 +197,11 @@ wrap_oserror2(space, e, w_name, exception_name='w_IOError', eintr_retry=True) - if not rposix._WIN32: - try: - _open_inhcache.set_non_inheritable(self.fd) - except OSError as e: - raise wrap_oserror2(space, e, w_name, - eintr_retry=False) + try: + _open_inhcache.set_non_inheritable(self.fd) + except OSError as e: + raise wrap_oserror2(space, e, w_name, + eintr_retry=False) else: w_fd = space.call_function(w_opener, w_name, space.newint(flags)) @@ -226,6 +225,7 @@ raise wrap_oserror2(space, e, w_name, eintr_retry=False) + try: st = os.fstat(self.fd) except OSError as e: diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -169,12 +169,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py --- a/pypy/module/_io/test/test_bytesio.py +++ b/pypy/module/_io/test/test_bytesio.py @@ -111,12 +111,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # b.close() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -173,12 +173,12 @@ # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, f.readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() @@ -288,31 +288,34 @@ raises(FileExistsError, _io.FileIO, filename, 'x') def test_non_inheritable(self): - import _io, posix + import _io + os = self.posix f = _io.FileIO(self.tmpfile, 'r') - assert posix.get_inheritable(f.fileno()) == False + assert os.get_inheritable(f.fileno()) == False f.close() def test_FileIO_fd_does_not_change_inheritable(self): - import _io, posix - fd1, fd2 = posix.pipe() - posix.set_inheritable(fd1, True) - posix.set_inheritable(fd2, False) + import _io + os = self.posix + fd1, fd2 = os.pipe() + os.set_inheritable(fd1, True) + os.set_inheritable(fd2, False) f1 = _io.FileIO(fd1, 'r') f2 = _io.FileIO(fd2, 'w') - assert posix.get_inheritable(fd1) == True - assert posix.get_inheritable(fd2) == False + assert os.get_inheritable(fd1) == True + assert os.get_inheritable(fd2) == False f1.close() f2.close() def test_close_upon_reinit(self): - import _io, posix + import _io + os = self.posix f = _io.FileIO(self.tmpfile, 'r') fd1 = f.fileno() f.__init__(self.tmpfile, 'w') fd2 = f.fileno() if fd1 != fd2: - raises(OSError, posix.close, fd1) + raises(OSError, os.close, fd1) def test_opener_negative(self): import _io diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -179,6 +179,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -191,14 +195,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -215,6 +218,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -259,8 +267,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -93,8 +93,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -610,13 +613,9 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyObject_GetBuffer', 'PyBuffer_Release', + 'PyObject_DelItemString', 'PyObject_GetBuffer', 'PyBuffer_Release', '_Py_setfilesystemdefaultencoding', - 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', - 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', - 'PyCObject_Type', '_Py_get_cobject_type', - 'PyCapsule_New', 'PyCapsule_IsValid', 'PyCapsule_GetPointer', 'PyCapsule_GetName', 'PyCapsule_GetDestructor', 'PyCapsule_GetContext', 'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor', @@ -648,6 +647,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree', 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', @@ -672,7 +672,7 @@ 'PyObject*', 'space.w_None', header=pypy_decl) register_global('_Py_TrueStruct', 'PyObject*', 'space.w_True', header=pypy_decl) -register_global('_Py_ZeroStruct', +register_global('_Py_FalseStruct', 'PyObject*', 'space.w_False', header=pypy_decl) register_global('_Py_NotImplementedStruct', 'PyObject*', 'space.w_NotImplemented', header=pypy_decl) @@ -716,8 +716,8 @@ "PyByteArray_Type": "space.w_bytearray", "PyMemoryView_Type": "space.w_memoryview", "PyBaseObject_Type": "space.w_object", - 'PyNone_Type': 'space.type(space.w_None)', - 'PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', + '_PyNone_Type': 'space.type(space.w_None)', + '_PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', 'PyCell_Type': 'space.gettypeobject(Cell.typedef)', 'PyModule_Type': 'space.gettypeobject(Module.typedef)', 'PyProperty_Type': 'space.gettypeobject(W_Property.typedef)', @@ -1121,9 +1121,6 @@ # of the cpyext module. The C functions are called with no wrapper, # but must not do anything like calling back PyType_Ready(). We # use them just to get a pointer to the PyTypeObjects defined in C. - get_cobject_type = rffi.llexternal('_%s_get_cobject_type' % prefix, - [], PyTypeObjectPtr, - compilation_info=eci, _nowrapper=True) get_capsule_type = rffi.llexternal('_%s_get_capsule_type' % prefix, [], PyTypeObjectPtr, compilation_info=eci, _nowrapper=True) @@ -1134,7 +1131,6 @@ def init_types(space): from pypy.module.cpyext.typeobject import py_type_ready from pypy.module.sys.interp_encoding import getfilesystemencoding - py_type_ready(space, get_cobject_type()) py_type_ready(space, get_capsule_type()) s = space.text_w(getfilesystemencoding(space)) setdefenc(rffi.str2charp(s, track_allocation=False)) # "leaks" @@ -1188,6 +1184,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1495,7 +1495,6 @@ source_dir / "pythonrun.c", source_dir / "sysmodule.c", source_dir / "complexobject.c", - source_dir / "cobject.c", source_dir / "structseq.c", source_dir / "capsule.c", source_dir / "pysignals.c", @@ -1549,7 +1548,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1563,6 +1561,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = ['#include ', + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, @@ -1673,7 +1674,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: @@ -1789,6 +1794,12 @@ return return exec_def(space, w_mod, mod_as_pyobj) +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -123,7 +123,6 @@ #include "memoryobject.h" #include "eval.h" #include "pymem.h" -#include "pycobject.h" #include "pycapsule.h" #include "bytesobject.h" #include "sliceobject.h" diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -4,6 +4,15 @@ extern "C" { #endif + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); + + /* + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ + + /* new buffer API */ #define PyObject_CheckBuffer(obj) \ @@ -27,6 +36,27 @@ /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ +/* Mapping protocol:*/ + + /* implemented as a macro: + + int PyMapping_DelItemString(PyObject *o, char *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItemString(O,K) PyObject_DelItemString((O),(K)) + + /* implemented as a macro: + + int PyMapping_DelItem(PyObject *o, PyObject *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -1,5 +1,4 @@ - -/* Bool object interface */ +/* Boolean object interface */ #ifndef Py_BOOLOBJECT_H #define Py_BOOLOBJECT_H @@ -7,15 +6,19 @@ extern "C" { #endif -#define Py_False ((PyObject *) &_Py_ZeroStruct) +#define PyBool_Check(x) (Py_TYPE(x) == &PyBool_Type) + +/* Py_False and Py_True are the only two bools in existence. +Don't forget to apply Py_INCREF() when returning either!!! */ + +/* Use these macros */ +#define Py_False ((PyObject *) &_Py_FalseStruct) #define Py_True ((PyObject *) &_Py_TrueStruct) /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False -#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) - #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/pycobject.h b/pypy/module/cpyext/include/pycobject.h deleted file mode 100644 --- a/pypy/module/cpyext/include/pycobject.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef Py_COBJECT_H -#define Py_COBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_DATA(PyTypeObject) PyCObject_Type; - -#define PyCObject_Check(op) ((op)->ob_type == &PyCObject_Type) - -/* Create a PyCObject from a pointer to a C object and an optional - destructor function. If the second argument is non-null, then it - will be called with the first argument if and when the PyCObject is - destroyed. - -*/ -PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtr( - void *cobj, void (*destruct)(void*)); - - -/* Create a PyCObject from a pointer to a C object, a description object, - and an optional destructor function. If the third argument is non-null, - then it will be called with the first and second arguments if and when - the PyCObject is destroyed. -*/ -PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtrAndDesc( - void *cobj, void *desc, void (*destruct)(void*,void*)); - -/* Retrieve a pointer to a C object from a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_AsVoidPtr(PyObject *); - -/* Retrieve a pointer to a description object from a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_GetDesc(PyObject *); - -/* Import a pointer to a C object from a module using a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_Import(const char *module_name, const char *cobject_name); - -/* Modify a C object. Fails (==0) if object has a destructor. */ -PyAPI_FUNC(int) PyCObject_SetVoidPtr(PyObject *self, void *cobj); - - -#if (PY_VERSION_HEX >= 0x02060000 || defined(Py_BUILD_CORE)) -typedef struct { - PyObject_HEAD - void *cobject; - void *desc; - void (*destructor)(void *); -} PyCObject; -#endif - -PyAPI_FUNC(PyTypeObject *) _Py_get_cobject_type(void); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_COBJECT_H */ diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -41,6 +41,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -41,19 +41,6 @@ w_mode = space.newtext(rffi.charp2str(mode)) return space.call_method(space.builtin, 'open', w_filename, w_mode) - at cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject) -def PyFile_FromFile(space, fp, name, mode, close): - """Create a new PyFileObject from the already-open standard C file - pointer, fp. The function close will be called when the file should be - closed. Return NULL on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.INT_real], lltype.Void) -def PyFile_SetBufSize(space, w_file, n): - """Available on systems with setvbuf() only. This should only be called - immediately after file object creation.""" - raise NotImplementedError - @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyFile_WriteString(space, s, w_p): """Write string s to file object p. Return 0 on success or -1 on @@ -75,9 +62,3 @@ w_str = space.repr(w_obj) space.call_method(w_p, "write", w_str) return 0 - - at cpython_api([PyObject], PyObject) -def PyFile_Name(space, w_p): - """Return the name of the file specified by p as a string object.""" - w_name = space.getattr(w_p, space.newtext("name")) - return w_name # borrowed ref, should be a W_StringObject from the file diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -23,6 +23,23 @@ /* Operations on any object */ int +PyObject_DelItemString(PyObject *o, char *key) +{ + PyObject *okey; + int ret; + + if (o == NULL || key == NULL) { + null_error(); + return -1; + } + okey = PyUnicode_FromString(key); + if (okey == NULL) + return -1; + ret = PyObject_DelItem(o, okey); + Py_DECREF(okey); + return ret; +} +int PyObject_CheckReadBuffer(PyObject *obj) { PyBufferProcs *pb = obj->ob_type->tp_as_buffer; @@ -96,6 +113,20 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) { @@ -111,6 +142,7 @@ return (void*)pointer; } + void _Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape) { @@ -253,19 +285,6 @@ -/* Buffer C-API for Python 3.0 */ - -int -PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) -{ - if (!PyObject_CheckBuffer(obj)) { - PyErr_Format(PyExc_TypeError, - "'%100s' does not have the buffer interface", - Py_TYPE(obj)->tp_name); - return -1; - } - return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); -} void PyBuffer_Release(Py_buffer *view) @@ -427,6 +446,7 @@ return retval; } + static PyObject * objargs_mktuple(va_list va) { diff --git a/pypy/module/cpyext/src/cobject.c b/pypy/module/cpyext/src/cobject.c deleted file mode 100644 --- a/pypy/module/cpyext/src/cobject.c +++ /dev/null @@ -1,162 +0,0 @@ - -/* Wrap void* pointers to be passed between C modules */ - -#include "Python.h" - - -/* Declarations for objects of type PyCObject */ - -typedef void (*destructor1)(void *); -typedef void (*destructor2)(void *, void*); - -PyObject * -PyCObject_FromVoidPtr(void *cobj, void (*destr)(void *)) -{ - PyCObject *self; - - self = PyObject_NEW(PyCObject, &PyCObject_Type); - if (self == NULL) - return NULL; - self->cobject=cobj; - self->destructor=destr; - self->desc=NULL; - - return (PyObject *)self; -} - -PyObject * -PyCObject_FromVoidPtrAndDesc(void *cobj, void *desc, - void (*destr)(void *, void *)) -{ - PyCObject *self; - - if (!desc) { - PyErr_SetString(PyExc_TypeError, - "PyCObject_FromVoidPtrAndDesc called with null" - " description"); - return NULL; - } - self = PyObject_NEW(PyCObject, &PyCObject_Type); - if (self == NULL) - return NULL; - self->cobject = cobj; - self->destructor = (destructor1)destr; - self->desc = desc; - - return (PyObject *)self; -} - -void * -PyCObject_AsVoidPtr(PyObject *self) -{ - if (self) { - if (PyCapsule_CheckExact(self)) { - const char *name = PyCapsule_GetName(self); - return (void *)PyCapsule_GetPointer(self, name); - } - if (self->ob_type == &PyCObject_Type) - return ((PyCObject *)self)->cobject; - PyErr_SetString(PyExc_TypeError, - "PyCObject_AsVoidPtr with non-C-object"); - } - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "PyCObject_AsVoidPtr called with null pointer"); - return NULL; -} - -void * -PyCObject_GetDesc(PyObject *self) -{ - if (self) { - if (self->ob_type == &PyCObject_Type) - return ((PyCObject *)self)->desc; - PyErr_SetString(PyExc_TypeError, - "PyCObject_GetDesc with non-C-object"); - } - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "PyCObject_GetDesc called with null pointer"); - return NULL; -} - -void * -PyCObject_Import(const char *module_name, const char *name) -{ - PyObject *m, *c; - void *r = NULL; - - if ((m = PyImport_ImportModule(module_name))) { - if ((c = PyObject_GetAttrString(m,name))) { - r = PyCObject_AsVoidPtr(c); - Py_DECREF(c); - } - Py_DECREF(m); - } - return r; -} - -int -PyCObject_SetVoidPtr(PyObject *self, void *cobj) -{ - PyCObject* cself = (PyCObject*)self; - if (cself == NULL || !PyCObject_Check(cself) || - cself->destructor != NULL) { - PyErr_SetString(PyExc_TypeError, - "Invalid call to PyCObject_SetVoidPtr"); - return 0; - } - cself->cobject = cobj; - return 1; -} - -static void -PyCObject_dealloc(PyCObject *self) -{ - if (self->destructor) { - if(self->desc) - ((destructor2)(self->destructor))(self->cobject, self->desc); - else - (self->destructor)(self->cobject); - } - PyObject_DEL(self); -} - - -PyDoc_STRVAR(PyCObject_Type__doc__, -"C objects to be exported from one extension module to another\n\ -\n\ -C objects are used for communication between extension modules. They\n\ -provide a way for an extension module to export a C interface to other\n\ -extension modules, so that extension modules can use the Python import\n\ -mechanism to link to one another."); - -PyTypeObject PyCObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "PyCObject", /*tp_name*/ - sizeof(PyCObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCObject_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - 0, /*tp_flags*/ - PyCObject_Type__doc__ /*tp_doc*/ -}; - -PyTypeObject *_Py_get_cobject_type(void) -{ - return &PyCObject_Type; -} diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,29 +1,10 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -CWCHARPP = lltype.Ptr(lltype.Array(rffi.CWCHARP, hints={'nolength': True})) - -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyModuleDef = rffi.VOIDP -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFrameObject = rffi.VOIDP -_inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP -struct_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype @cpython_api([rffi.CCHARP], Py_ssize_t, error=-1) @@ -228,39 +209,6 @@ this method returns zero and sets errno to EDOM.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def PyOS_stricmp(space, s1, s2): """Case insensitive comparison of strings. The function works almost @@ -275,24 +223,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -483,31 +413,6 @@ 0 on success, -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=-1) -def Py_ReprEnter(space, object): - """Called at the beginning of the tp_repr implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the tp_repr implementation - should return a string object indicating a cycle. As examples, - dict objects return {...} and list objects - return [...]. - - The function will return a negative integer if the recursion limit - is reached. In that case the tp_repr implementation should - typically return NULL. - - Otherwise, the function returns zero and the tp_repr - implementation can continue normally.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_ReprLeave(space, object): - """Ends a Py_ReprEnter(). Must be called once for each - invocation of Py_ReprEnter() that returns zero.""" - raise NotImplementedError - @cpython_api([rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFd(space, fd, name, mode, buffering, encoding, errors, newline, closefd): """Create a Python file object from the file descriptor of an already @@ -1295,39 +1200,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.VOIDP) -def PyModule_GetState(space, module): - """Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or NULL. See - PyModuleDef.m_size.""" - raise NotImplementedError - - at cpython_api([PyObject], PyModuleDef) -def PyModule_GetDef(space, module): - """Return a pointer to the PyModuleDef struct from which the module was - created, or NULL if the module wasn't created with - PyModule_Create().""" - raise NotImplementedError - - - at cpython_api([PyModuleDef], PyObject) -def PyModule_Create(space, module): - """Create a new module object, given the definition in module. This behaves - like PyModule_Create2() with module_api_version set to - PYTHON_API_VERSION.""" - raise NotImplementedError - - - at cpython_api([PyModuleDef, rffi.INT_real], PyObject) -def PyModule_Create2(space, module, module_api_version): - """Create a new module object, given the definition in module, assuming the - API version module_api_version. If that version does not match the version - of the running interpreter, a RuntimeWarning is emitted. - - Most uses of this function should be using PyModule_Create() - instead; only use this if you are sure you need it.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.INT_real], PyObject) def PyNumber_ToBase(space, n, base): """Returns the integer n converted to base base as a string. The base @@ -1337,23 +1209,6 @@ PyNumber_Index() first.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_Bytes(space, o): - """ - Compute a bytes representation of object o. NULL is returned on - failure and a bytes object on success. This is equivalent to the Python - expression bytes(o), when o is not an integer. Unlike bytes(o), - a TypeError is raised when o is an integer instead of a zero-initialized - bytes object.""" - raise NotImplementedError - - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - raise NotImplementedError - borrow_from() - @cpython_api([PyFrameObject], rffi.INT_real, error=-1) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -510,3 +510,15 @@ assert run_async(list1()) == ([], [0, 1, 2]) assert run_async(list2()) == ([], [0, 1, 2]) """ + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyLong_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/cpyext/test/test_pycobject.py b/pypy/module/cpyext/test/test_pycobject.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_pycobject.py +++ /dev/null @@ -1,30 +0,0 @@ -import py -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - -class AppTestStringObject(AppTestCpythonExtensionBase): - def test_pycobject_import(self): - module = self.import_extension('foo', [ - ("set_ptr", "METH_O", - """ - PyObject *pointer, *module; - void *ptr = PyLong_AsVoidPtr(args); - if (PyErr_Occurred()) return NULL; - pointer = PyCObject_FromVoidPtr(ptr, NULL); - if (PyErr_Occurred()) return NULL; - module = PyImport_ImportModule("foo"); - PyModule_AddObject(module, "_ptr", pointer); - Py_DECREF(module); - if (PyErr_Occurred()) return NULL; - Py_RETURN_NONE; - """), - ("get_ptr", "METH_NOARGS", - """ - void *ptr = PyCObject_Import("foo", "_ptr"); - if (PyErr_Occurred()) return NULL; - return PyLong_FromVoidPtr(ptr); - """)]) - module.set_ptr(1234) - assert "PyCObject object" in str(module._ptr) - import gc; gc.collect() - assert module.get_ptr() == 1234 - del module._ptr diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -48,17 +48,6 @@ space.call_method(w_file, "close") - def test_file_name(self, space, api): - name = str(udir / "_test_file") - with rffi.scoped_str2charp(name) as filename: - with rffi.scoped_str2charp("wb") as mode: - w_file = api.PyFile_FromString(filename, mode) - assert space.str_w(api.PyFile_Name(w_file)) == name - - @pytest.mark.xfail - def test_file_setbufsize(self, space, api): - api.PyFile_SetBufSize() - def test_file_writestring(self, space, api, capfd): w_stdout = space.sys.get("stdout") with rffi.scoped_str2charp("test\n") as s: diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -677,13 +677,15 @@ def test_decode_null_encoding(self, space): null_charp = lltype.nullptr(rffi.CCHARP.TO) u_text = u'abcdefg' - s_text = space.str_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp)) + s_text = space.bytes_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp)) b_text = rffi.str2charp(s_text) assert space.unicode_w(PyUnicode_Decode( space, b_text, len(s_text), null_charp, null_charp)) == u_text with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.newbytes(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -486,11 +486,16 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" - if not encoding: - # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead - encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) - w_encoding = space.newtext(rffi.charp2str(encoding)) + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): + if encoding: + w_encoding = space.newtext(rffi.charp2str(encoding)) + else: + # python 3.4 changed to this from defaultencoding + w_encoding = space.newtext('utf-8') + w_str = space.newbytes(s) if errors: w_errors = space.newtext(rffi.charp2str(errors)) else: @@ -525,28 +530,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + raise oefmt(space.w_TypeError, "decoding str is not supported") + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.charbuf_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -232,9 +232,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print(type(st).__module__) + # print(type(st).__module__) s = pickle.dumps(st) - print(repr(s)) + # print(repr(s)) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -578,6 +578,12 @@ res = fp.read() assert res == '1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): # nt specific diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -231,11 +231,13 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -271,7 +273,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -289,11 +290,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -49,7 +49,8 @@ # Run signal handlers if we were interrupted space.getexecutioncontext().checksignals() if microseconds >= 0: - microseconds = r_longlong(endtime - (time.time() * 1e6)) + microseconds = r_longlong((endtime - (time.time() * 1e6)) + + 0.999) # Check for negative values, since those mean block # forever if microseconds <= 0: @@ -247,7 +248,7 @@ "cannot release un-acquired lock") self.rlock_count -= 1 if self.rlock_count == 0: - self.rlock_owner == 0 + self.rlock_owner = 0 self.lock.release() def is_owned_w(self, space): diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -306,6 +306,9 @@ finally: signal.signal(signal.SIGALRM, oldalrm) + +class AppTestLockRepr(GenericTestThread): + def test_lock_repr(self): import _thread lock = _thread.allocate_lock() diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -151,7 +151,7 @@ if getattr(func, '_elidable_function_', False): raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) if not getattr(func, '_jit_look_inside_', True): - raise TypeError("it does not make sense for %s to be both elidable and dont_look_inside" % func) + raise TypeError("it does not make sense for %s to be both unroll_safe and dont_look_inside" % func) func._jit_unroll_safe_ = True return func diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,25 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -730,16 +734,21 @@ length = rwin32.MAX_PATH + 1 traits = _preferred_traits(path) win32traits = make_win32_traits(traits) - with traits.scoped_alloc_buffer(length) as buf: - res = win32traits.GetFullPathName( - traits.as_str0(path), rffi.cast(rwin32.DWORD, length), - buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) - if res == 0: - raise rwin32.lastSavedWindowsError("_getfullpathname failed") - result = buf.str(intmask(res)) - assert result is not None - result = rstring.assert_str0(result) - return result + while True: # should run the loop body maximum twice + with traits.scoped_alloc_buffer(length) as buf: + res = win32traits.GetFullPathName( + traits.as_str0(path), rffi.cast(rwin32.DWORD, length), + buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) + res = intmask(res) + if res == 0: + raise rwin32.lastSavedWindowsError("_getfullpathname failed") + if res >= length: + length = res + 1 + continue + result = buf.str(res) + assert result is not None + result = rstring.assert_str0(result) + return result c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP, diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1775,8 +1775,6 @@ "truncated input", s, pos, size) result.append(res) - if pos > size - unicode_bytes: - break continue t = r_uint(0) h = 0 diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,12 +196,28 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, NULL_HANDLE, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -215,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] From pypy.commits at gmail.com Sun Nov 18 03:57:42 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 00:57:42 -0800 (PST) Subject: [pypy-commit] pypy py3.6: copy _PyErr_FormatFromCause from CPython Message-ID: <5bf12986.1c69fb81.983f7.a316@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95330:d00dd348bccc Date: 2018-11-18 00:15 -0800 http://bitbucket.org/pypy/pypy/changeset/d00dd348bccc/ Log: copy _PyErr_FormatFromCause from CPython diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -606,7 +606,7 @@ '_Py_BuildValue_SizeT', '_Py_VaBuildValue_SizeT', 'PyErr_Format', 'PyErr_NewException', 'PyErr_NewExceptionWithDoc', - 'PyErr_WarnFormat', + 'PyErr_WarnFormat', '_PyErr_FormatFromCause', 'PySys_WriteStdout', 'PySys_WriteStderr', 'PyEval_CallFunction', 'PyEval_CallMethod', 'PyObject_CallFunction', diff --git a/pypy/module/cpyext/src/pyerrors.c b/pypy/module/cpyext/src/pyerrors.c --- a/pypy/module/cpyext/src/pyerrors.c +++ b/pypy/module/cpyext/src/pyerrors.c @@ -20,7 +20,42 @@ return NULL; } +PyObject * +_PyErr_FormatFromCause(PyObject *exception, const char *format, ...) +{ + PyObject *exc, *val, *val2, *tb; + va_list vargs; + PyObject* string; +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + assert(PyErr_Occurred()); + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + assert(!PyErr_Occurred()); + + string = PyUnicode_FromFormatV(format, vargs); + PyErr_SetObject(exception, string); + Py_XDECREF(string); + va_end(vargs); + + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); + + return NULL; +} PyObject * PyErr_NewException(const char *name, PyObject *base, PyObject *dict) From pypy.commits at gmail.com Sun Nov 18 03:57:44 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 00:57:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: put common case first, exit quickly on empty byte string Message-ID: <5bf12988.1c69fb81.14e11.1136@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95331:0d38f5b5e3e5 Date: 2018-11-18 00:55 -0800 http://bitbucket.org/pypy/pypy/changeset/0d38f5b5e3e5/ Log: put common case first, exit quickly on empty byte string diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -530,11 +530,16 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if space.isinstance_w(w_obj, space.w_unicode): + if space.isinstance_w(w_obj, space.w_bytes): + s = space.bytes_w(w_obj) + if not s: + return space.newtext('', 0) + elif space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") - s = space.charbuf_w(w_obj) + else: + s = space.buffer_w(w_obj, 0) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Sun Nov 18 03:57:47 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 00:57:47 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 into branch Message-ID: <5bf1298b.1c69fb81.bc0de.bf87@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95332:fb5a5a9969b0 Date: 2018-11-18 00:56 -0800 http://bitbucket.org/pypy/pypy/changeset/fb5a5a9969b0/ Log: merge py3.5 into branch diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1073,6 +1073,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -42,7 +42,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -75,7 +75,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -138,11 +138,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -156,7 +156,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print("Called", self.num) -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print(sys.executable, self.tmpfile) - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -68,18 +68,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = io.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = io.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -530,11 +530,16 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if space.isinstance_w(w_obj, space.w_unicode): + if space.isinstance_w(w_obj, space.w_bytes): + s = space.bytes_w(w_obj) + if not s: + return space.newtext('', 0) + elif space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") - s = space.charbuf_w(w_obj) + else: + s = space.buffer_w(w_obj, 0) return _pyunicode_decode(space, s, encoding, errors) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): From pypy.commits at gmail.com Sun Nov 18 14:00:33 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 11:00:33 -0800 (PST) Subject: [pypy-commit] pypy py3.6: fix typo (using unicode-utf8 syntax too early) Message-ID: <5bf1b6d1.1c69fb81.3440e.b268@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95333:27c28f481460 Date: 2018-11-18 10:59 -0800 http://bitbucket.org/pypy/pypy/changeset/27c28f481460/ Log: fix typo (using unicode-utf8 syntax too early) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -533,7 +533,7 @@ if space.isinstance_w(w_obj, space.w_bytes): s = space.bytes_w(w_obj) if not s: - return space.newtext('', 0) + return space.newtext('') elif space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific From pypy.commits at gmail.com Sun Nov 18 21:58:49 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 18:58:49 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix typo (using unicode-utf8 syntax too early) Message-ID: <5bf226e9.1c69fb81.b790.11e9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95334:064564894563 Date: 2018-11-18 10:59 -0800 http://bitbucket.org/pypy/pypy/changeset/064564894563/ Log: fix typo (using unicode-utf8 syntax too early) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -533,7 +533,7 @@ if space.isinstance_w(w_obj, space.w_bytes): s = space.bytes_w(w_obj) if not s: - return space.newtext('', 0) + return space.newtext('') elif space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific From pypy.commits at gmail.com Sun Nov 18 22:36:35 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:35 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: refactor builting erro handlers to use utf8 indices, add failing test Message-ID: <5bf22fc3.1c69fb81.1b3db.6472@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95335:de06359bbf5c Date: 2018-11-16 12:28 -0800 http://bitbucket.org/pypy/pypy/changeset/de06359bbf5c/ Log: refactor builting erro handlers to use utf8 indices, add failing test diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -379,25 +379,23 @@ def surrogatepass_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - utf8 = space.utf8_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) encoding = space.text_w(space.getattr(w_exc, space.newtext('encoding'))) - msg = space.text_w(space.getattr(w_exc, space.newtext('reason'))) bytelength, code = get_standard_encoding(encoding) if code == ENC_UNKNOWN: # Not supported, fail with original exception raise OperationError(space.type(w_exc), w_exc) end = space.int_w(w_end) builder = StringBuilder() + start = w_obj._index_to_byte(start) + end = w_obj._index_to_byte(end) + obj = w_obj._utf8 pos = start - # start, end are in codepoint indices - itr = rutf8.Utf8StringIterator(utf8) - for i in range(pos): - itr.next() while pos < end: - ch = itr.next() - pos += 1 + ch = rutf8.codepoint_at_pos(obj, pos) + pos = rutf8.next_codepoint_pos(obj, pos) if ch < 0xd800 or ch > 0xdfff: # Not a surrogate, fail with original exception raise OperationError(space.type(w_exc), w_exc) @@ -465,22 +463,22 @@ def surrogateescape_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - utf8 = space.utf8_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) res = '' + start = w_obj._index_to_byte(start) + end = w_obj._index_to_byte(end) + obj = w_obj._utf8 pos = start - itr = rutf8.Utf8StringIterator(utf8) - for i in range(pos): - itr.next() while pos < end: - ch = itr.next() - pos += 1 - if ch < 0xdc80 or ch > 0xdcff: + code = rutf8.codepoint_at_pos(obj, pos) + if code < 0xdc80 or code > 0xdcff: # Not a UTF-8b surrogate, fail with original exception raise OperationError(space.type(w_exc), w_exc) - res += chr(ch - 0xdc00) + res += chr(code - 0xdc00) + pos = rutf8.next_codepoint_pos(obj, pos) return space.newtuple([space.newbytes(res), w_end]) elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError): consumed = 0 diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -625,6 +625,8 @@ assert '[\uDC80]'.encode('utf-8', 'namereplace') == b'[\\udc80]' def test_surrogateescape(self): + assert "\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", + "surrogateescape") == b"\xe4\xeb\xef\xf6\xfc" assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b' assert 'a\udc80b'.encode('utf-8', 'surrogateescape') == b'a\x80b' for enc in ('utf-8', 'ascii', 'latin-1', 'charmap'): From pypy.commits at gmail.com Sun Nov 18 22:36:37 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:37 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: try one approach Message-ID: <5bf22fc5.1c69fb81.f5955.d25f@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95336:28269456b8aa Date: 2018-11-16 12:28 -0800 http://bitbucket.org/pypy/pypy/changeset/28269456b8aa/ Log: try one approach diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1201,8 +1201,6 @@ def encode_object(space, w_object, encoding, errors, allow_surrogates=False): - # TODO: refactor unnatrual use of error hanlders here, - # we should make a single pass over the utf8 str from pypy.module._codecs.interp_codecs import encode_text, CodecState utf8 = space.utf8_w(w_object) if not allow_surrogates: @@ -1212,23 +1210,22 @@ state = space.fromcache(CodecState) eh = state.encode_error_handler if pos >= 0: + # remove surrogates in pieces, eh needs codepoint positions + res = [] while pos >= 0: - start = utf8[:pos] upos = rutf8.codepoints_in_utf8(utf8, end=pos) - ru, _pos = eh(errors, "utf8", "surrogates not allowed", utf8, + ru, _pos = eh(errors, encoding, "surrogates not allowed", utf8, upos, upos + 1) - upos = rutf8.next_codepoint_pos(utf8, _pos) - end = utf8[upos:] - utf8 = start + ru + end - _pos = rutf8.surrogate_in_utf8(utf8) - if _pos <= pos: - # surrogatepass? - break - pos = _pos - if errors == 'surrogateescape': - #escape - return space.newbytes(utf8) + res.append(utf8[:pos]) + res.append(ru) + utf8_pos = rutf8.next_codepoint_pos(utf8, _pos) + utf8 = utf8[utf8_pos:] + pos = rutf8.surrogate_in_utf8(utf8) + res.append(utf8) + utf8 = ''.join(res) w_object = space.newtext(utf8) + # change the errors to only do the encoding now + errors = 'strict' if errors is None or errors == 'strict': if encoding is None or encoding == 'utf-8': #if rutf8.has_surrogates(utf8): @@ -1243,7 +1240,6 @@ a.pos, a.pos + 1) assert False, "always raises" return space.newbytes(utf8) - if encoding is None: encoding = space.sys.defaultencoding w_retval = encode_text(space, w_object, encoding, errors) From pypy.commits at gmail.com Sun Nov 18 22:36:39 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:39 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: return value from encode errorhandler is bytes, not unicode Message-ID: <5bf22fc7.1c69fb81.6e90.0857@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95337:6eeefdec5c2e Date: 2018-11-18 01:04 -0800 http://bitbucket.org/pypy/pypy/changeset/6eeefdec5c2e/ Log: return value from encode errorhandler is bytes, not unicode diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -277,11 +277,11 @@ msg = "ordinal not in range(256)" res_8, newindex = errorhandler( errors, 'latin1', msg, s, startindex, index) - for cp in rutf8.Utf8StringIterator(res_8): - if cp > 0xFF: + for ch in res_8: + if ord(ch) > 0xFF: errorhandler("strict", 'latin1', msg, s, startindex, index) raise RuntimeError('error handler should not have returned') - result.append(chr(cp)) + result.append(ch) if index != newindex: # Should be uncommon index = newindex pos = rutf8._pos_at_index(s, newindex) From pypy.commits at gmail.com Sun Nov 18 22:36:40 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:40 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: simplify andfix overly-engineered bogus code Message-ID: <5bf22fc8.1c69fb81.7f50.5e5f@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95338:636e47b82bce Date: 2018-11-18 01:05 -0800 http://bitbucket.org/pypy/pypy/changeset/636e47b82bce/ Log: simplify andfix overly-engineered bogus code diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -191,9 +191,8 @@ def descr_decode(self, space, w_encoding=None, w_errors=None): from pypy.objspace.std.unicodeobject import ( - _get_encoding_and_errors, decode_object) - encoding, errors, allow_surrogates = _get_encoding_and_errors(space, - w_encoding, w_errors) + get_encoding_and_errors, decode_object) + encoding, errors = get_encoding_and_errors(space, w_encoding, w_errors) if errors is None: errors = 'strict' if encoding is None: diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -189,7 +189,7 @@ if w_object is None: w_value = W_UnicodeObject.EMPTY else: - encoding, errors, allow_surrogates = _get_encoding_and_errors(space, + encoding, errors = get_encoding_and_errors(space, w_encoding, w_errors) if encoding is None and errors is None: # this is very quick if w_object is already a w_unicode @@ -522,10 +522,8 @@ return space.w_True def descr_encode(self, space, w_encoding=None, w_errors=None): - encoding, errors, allow_surrogates = _get_encoding_and_errors(space, - w_encoding, w_errors) - return encode_object(space, self, encoding, errors, - allow_surrogates=allow_surrogates) + encoding, errors = get_encoding_and_errors(space, w_encoding, w_errors) + return encode_object(space, self, encoding, errors) @unwrap_spec(tabsize=int) def descr_expandtabs(self, space, tabsize=8): @@ -1191,42 +1189,16 @@ return space.sys.defaultencoding -def _get_encoding_and_errors(space, w_encoding, w_errors): +def get_encoding_and_errors(space, w_encoding, w_errors): encoding = None if w_encoding is None else space.text_w(w_encoding) errors = None if w_errors is None else space.text_w(w_errors) - allow_surrogates = False - if encoding and 'escape' in encoding: - allow_surrogates = True - return encoding, errors, allow_surrogates + return encoding, errors -def encode_object(space, w_object, encoding, errors, allow_surrogates=False): +def encode_object(space, w_object, encoding, errors): from pypy.module._codecs.interp_codecs import encode_text, CodecState - utf8 = space.utf8_w(w_object) - if not allow_surrogates: - if errors is None: - errors = 'strict' - pos = rutf8.surrogate_in_utf8(utf8) - state = space.fromcache(CodecState) - eh = state.encode_error_handler - if pos >= 0: - # remove surrogates in pieces, eh needs codepoint positions - res = [] - while pos >= 0: - upos = rutf8.codepoints_in_utf8(utf8, end=pos) - ru, _pos = eh(errors, encoding, "surrogates not allowed", utf8, - upos, upos + 1) - res.append(utf8[:pos]) - res.append(ru) - utf8_pos = rutf8.next_codepoint_pos(utf8, _pos) - utf8 = utf8[utf8_pos:] - pos = rutf8.surrogate_in_utf8(utf8) - res.append(utf8) - utf8 = ''.join(res) - w_object = space.newtext(utf8) - # change the errors to only do the encoding now - errors = 'strict' if errors is None or errors == 'strict': + utf8 = space.utf8_w(w_object) if encoding is None or encoding == 'utf-8': #if rutf8.has_surrogates(utf8): # utf8 = rutf8.reencode_utf8_with_surrogates(utf8) From pypy.commits at gmail.com Sun Nov 18 22:36:42 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:42 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: distinguish between errorhandler returning unicode and bytes Message-ID: <5bf22fca.1c69fb81.da91.228a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95339:9dc3de0b34d5 Date: 2018-11-18 18:36 -0800 http://bitbucket.org/pypy/pypy/changeset/9dc3de0b34d5/ Log: distinguish between errorhandler returning unicode and bytes diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -27,7 +27,7 @@ def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos): assert startingpos >= 0 ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]] - return ''.join(ux), endingpos + return ''.join(ux), endingpos, 'b' @specialize.memo() def encode_error_handler(space): @@ -199,7 +199,7 @@ while i < len(s): ch = s[i] if ord(ch) > 0x7F: - r, i = errorhandler(errors, 'ascii', 'ordinal not in range(128)', + r, i, rettype = errorhandler(errors, 'ascii', 'ordinal not in range(128)', s, i, i + 1) res.append(r) else: @@ -242,7 +242,7 @@ assert pos >= 0 start = s[:pos] upos = rutf8.codepoints_in_utf8(s, end=pos) - ru, lgt = errorhandler(errors, 'utf8', + ru, lgt, rettype = errorhandler(errors, 'utf8', 'surrogates not allowed', s, upos, upos + 1) end = utf8_encode_utf_8(s[pos+3:], errors, errorhandler, allow_surrogates=allow_surrogates) @@ -275,13 +275,20 @@ pos = rutf8.next_codepoint_pos(s, pos) index += 1 msg = "ordinal not in range(256)" - res_8, newindex = errorhandler( + res, newindex, rettype = errorhandler( errors, 'latin1', msg, s, startindex, index) - for ch in res_8: - if ord(ch) > 0xFF: - errorhandler("strict", 'latin1', msg, s, startindex, index) - raise RuntimeError('error handler should not have returned') - result.append(ch) + if rettype == 'u': + for cp in rutf8.Utf8StringIterator(res): + if cp > 0xFF: + errorhandler("strict", 'latin1', msg, s, startindex, index) + raise RuntimeError('error handler should not have returned') + result.append(chr(cp)) + else: + for ch in res: + if ord(ch) > 0xFF: + errorhandler("strict", 'latin1', msg, s, startindex, index) + raise RuntimeError('error handler should not have returned') + result.append(ch) if index != newindex: # Should be uncommon index = newindex pos = rutf8._pos_at_index(s, newindex) @@ -309,9 +316,20 @@ pos = rutf8.next_codepoint_pos(s, pos) index += 1 msg = "ordinal not in range(128)" - res_8, newindex = errorhandler( + res, newindex, rettype = errorhandler( errors, 'ascii', msg, s, startindex, index) - result.append(res_8) + if rettype == 'u': + for cp in rutf8.Utf8StringIterator(res): + if cp > 0x80: + errorhandler("strict", 'ascii', msg, s, startindex, index) + raise RuntimeError('error handler should not have returned') + result.append(chr(cp)) + else: + for ch in res: + if ord(ch) > 0x80: + errorhandler("strict", 'ascii', msg, s, startindex, index) + raise RuntimeError('error handler should not have returned') + result.append(ch) pos = rutf8._pos_at_index(s, newindex) return result.build() @@ -346,7 +364,7 @@ continue if ordch1 <= 0xC1: - r, pos = errorhandler(errors, "utf8", "invalid start byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid start byte", s, pos, pos + 1) res.append(r) continue @@ -358,14 +376,14 @@ if not final: pos -= 1 break - r, pos = errorhandler(errors, "utf8", "unexpected end of data", + r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", s, pos - 1, pos) res.append(r) continue ordch2 = ord(s[pos]) if rutf8._invalid_byte_2_of_2(ordch2): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) res.append(r) continue @@ -380,7 +398,7 @@ if not final: pos -= 1 break - r, pos = errorhandler(errors, "utf8", "unexpected end of data", + r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", s, pos - 1, pos + 1) res.append(r) continue @@ -388,12 +406,12 @@ ordch3 = ord(s[pos + 1]) if rutf8._invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) res.append(r) continue elif rutf8._invalid_byte_3_of_3(ordch3): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos + 1) res.append(r) continue @@ -410,26 +428,25 @@ if not final: pos -= 1 break - r, pos = errorhandler(errors, "utf8", "unexpected end of data", + r, pos, rettype = errorhandler(errors, "utf8", "unexpected end of data", s, pos - 1, pos) - res.append(r) continue ordch2 = ord(s[pos]) ordch3 = ord(s[pos + 1]) ordch4 = ord(s[pos + 2]) if rutf8._invalid_byte_2_of_4(ordch1, ordch2): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos) res.append(r) continue elif rutf8._invalid_byte_3_of_4(ordch3): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos + 1) res.append(r) continue elif rutf8._invalid_byte_4_of_4(ordch4): - r, pos = errorhandler(errors, "utf8", "invalid continuation byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid continuation byte", s, pos - 1, pos + 2) res.append(r) continue @@ -442,7 +459,7 @@ res.append(chr(ordch4)) continue - r, pos = errorhandler(errors, "utf8", "invalid start byte", + r, pos, rettype = errorhandler(errors, "utf8", "invalid start byte", s, pos - 1, pos) res.append(r) @@ -458,9 +475,9 @@ endinpos = pos while endinpos < len(s) and s[endinpos] in hexdigits: endinpos += 1 - res, pos = errorhandler( + r, pos, rettype = errorhandler( errors, encoding, message, s, pos - 2, endinpos) - builder.append(res) + builder.append(r) else: try: chr = int(s[pos:pos + digits], 16) @@ -468,9 +485,9 @@ endinpos = pos while s[endinpos] in hexdigits: endinpos += 1 - res, pos = errorhandler( + r, pos, rettype = errorhandler( errors, encoding, message, s, pos - 2, endinpos) - builder.append(res) + builder.append(r) else: # when we get here, chr is a 32-bit unicode character try: @@ -478,9 +495,9 @@ pos += digits except ValueError: message = "illegal Unicode character" - res, pos = errorhandler( + r, pos, rettype = errorhandler( errors, encoding, message, s, pos - 2, pos + digits) - builder.append(res) + builder.append(r) return pos def str_decode_unicode_escape(s, errors, final, errorhandler, ud_handler): @@ -506,9 +523,9 @@ pos += 1 if pos >= size: message = "\\ at end of string" - res, pos = errorhandler(errors, "unicodeescape", + r, pos, rettype = errorhandler(errors, "unicodeescape", message, s, pos - 1, size) - builder.append(res) + builder.append(r) continue ch = s[pos] @@ -586,21 +603,21 @@ name = s[pos + 1:look] code = ud_handler.call(name) if code < 0: - res, pos = errorhandler( + r, pos, rettype = errorhandler( errors, "unicodeescape", message, s, pos - 1, look + 1) - builder.append(res) + builder.append(r) continue pos = look + 1 builder.append_code(code) else: - res, pos = errorhandler(errors, "unicodeescape", + r, pos, rettype = errorhandler(errors, "unicodeescape", message, s, pos - 1, look + 1) - builder.append(res) + builder.append(r) else: - res, pos = errorhandler(errors, "unicodeescape", + r, pos, rettype = errorhandler(errors, "unicodeescape", message, s, pos - 1, look + 1) - builder.append(res) + builder.append(r) else: builder.append_char('\\') builder.append_code(ord(ch)) @@ -867,22 +884,22 @@ # We've seen at least one base-64 character pos += 1 msg = "partial character in shift sequence" - res, pos = errorhandler(errors, 'utf7', + r, pos, rettype = errorhandler(errors, 'utf7', msg, s, pos-1, pos) - reslen = rutf8.check_utf8(res, True) + reslen = rutf8.check_utf8(r, True) outsize += reslen - result.append(res) + result.append(r) continue else: # Some bits remain; they should be zero if base64buffer != 0: pos += 1 msg = "non-zero padding bits in shift sequence" - res, pos = errorhandler(errors, 'utf7', + r, pos, rettype = errorhandler(errors, 'utf7', msg, s, pos-1, pos) - reslen = rutf8.check_utf8(res, True) + reslen = rutf8.check_utf8(r, True) outsize += reslen - result.append(res) + result.append(r) continue if surrogate and _utf7_DECODE_DIRECT(ord(ch)): @@ -917,10 +934,10 @@ startinpos = pos pos += 1 msg = "unexpected special character" - res, pos = errorhandler(errors, 'utf7', msg, s, pos-1, pos) - reslen = rutf8.check_utf8(res, True) + r, pos, rettype = errorhandler(errors, 'utf7', msg, s, pos-1, pos) + reslen = rutf8.check_utf8(r, True) outsize += reslen - result.append(res) + result.append(r) # end of string final_length = result.getlength() @@ -931,10 +948,10 @@ base64bits >= 6 or (base64bits > 0 and base64buffer != 0)): msg = "unterminated shift sequence" - res, pos = errorhandler(errors, 'utf7', msg, s, shiftOutStartPos, pos) - reslen = rutf8.check_utf8(res, True) + r, pos, rettype = errorhandler(errors, 'utf7', msg, s, shiftOutStartPos, pos) + reslen = rutf8.check_utf8(r, True) outsize += reslen - result.append(res) + result.append(r) final_length = result.getlength() elif inShift: pos = startinpos @@ -1101,7 +1118,7 @@ if len(s) - pos < 2: if not final: break - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "truncated data", s, pos, len(s)) result.append(r) @@ -1118,7 +1135,7 @@ if not final: break errmsg = "unexpected end of data" - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, errmsg, s, pos, len(s)) result.append(r) if len(s) - pos < 2: @@ -1131,12 +1148,12 @@ rutf8.unichr_as_utf8_append(result, ch) continue else: - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "illegal UTF-16 surrogate", s, pos - 4, pos - 2) result.append(r) else: - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "illegal encoding", s, pos - 2, pos) result.append(r) @@ -1176,44 +1193,62 @@ index = 0 while pos < size: try: - ch = rutf8.codepoint_at_pos(s, pos) + cp = rutf8.codepoint_at_pos(s, pos) except IndexError: # malformed codepoint, blindly use ch - ch = ord(s[pos]) pos += 1 if errorhandler: - res_8, newindex = errorhandler( + r, newindex, rettype = errorhandler( errors, public_encoding_name, 'malformed unicode', s, pos - 1, pos) - for cp in rutf8.Utf8StringIterator(res_8): - if cp < 0xD800: + if rettype == 'u': + for cp in rutf8.Utf8StringIterator(r): + if cp < 0xD800: + _STORECHAR(result, cp, byteorder) + else: + errorhandler('strict', public_encoding_name, + 'malformed unicode', + s, pos-1, pos) + else: + for ch in r: + cp = ord(ch) + if cp < 0xD800: + _STORECHAR(result, cp, byteorder) + else: + errorhandler('strict', public_encoding_name, + 'malformed unicode', + s, pos-1, pos) + else: + cp = ord(s[pos]) + _STORECHAR(result, cp, byteorder) + continue + if cp < 0xD800: + _STORECHAR(result, cp, byteorder) + elif cp >= 0x10000: + _STORECHAR(result, 0xD800 | ((cp-0x10000) >> 10), byteorder) + _STORECHAR(result, 0xDC00 | ((cp-0x10000) & 0x3FF), byteorder) + elif cp >= 0xE000 or allow_surrogates: + _STORECHAR(result, cp, byteorder) + else: + r, newindex, rettype = errorhandler( + errors, public_encoding_name, 'surrogates not allowed', + s, pos, pos+1) + if rettype == 'u': + for cp in rutf8.Utf8StringIterator(r): + if cp < 0xD800 or allow_surrogates: _STORECHAR(result, cp, byteorder) else: errorhandler('strict', public_encoding_name, - 'malformed unicode', - s, pos-1, pos) + 'surrogates not allowed', + s, pos, pos+1) else: - _STORECHAR(result, ch, byteorder) - continue - if ch < 0xD800: - _STORECHAR(result, ch, byteorder) - elif ch >= 0x10000: - _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder) - _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder) - elif ch >= 0xE000 or allow_surrogates: - _STORECHAR(result, ch, byteorder) - else: - res_8, newindex = errorhandler( - errors, public_encoding_name, 'surrogates not allowed', - s, pos, pos+1) - #for cp in rutf8.Utf8StringIterator(res_8): - for ch in res_8: - cp = ord(ch) - if cp < 0xD800 or allow_surrogates: - _STORECHAR(result, cp, byteorder) - else: - errorhandler('strict', public_encoding_name, - 'surrogates not allowed', + for ch in r: + cp = ord(ch) + if cp < 0xD800 or allow_surrogates: + _STORECHAR(result, cp, byteorder) + else: + errorhandler('strict', public_encoding_name, + 'surrogates not allowed', s, pos, pos+1) if index != newindex: # Should be uncommon index = newindex @@ -1329,7 +1364,7 @@ if len(s) - pos < 4: if not final: break - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "truncated data", s, pos, len(s)) result.append(r) @@ -1339,14 +1374,14 @@ ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) | (ord(s[pos + iorder[1]]) << 8) | ord(s[pos + iorder[0]])) if not allow_surrogates and 0xD800 <= ch <= 0xDFFF: - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "code point in surrogate code point " "range(0xd800, 0xe000)", s, pos, pos + 4) result.append(r) continue elif ch >= 0x110000: - r, pos = errorhandler(errors, public_encoding_name, + r, pos, rettype = errorhandler(errors, public_encoding_name, "codepoint not in range(0x110000)", s, pos, len(s)) result.append(r) @@ -1404,11 +1439,20 @@ ch = ord(s[pos]) pos += 1 if errorhandler: - res_8, newindex = errorhandler( + r, newindex, rettype = errorhandler( errors, public_encoding_name, 'malformed unicode', s, index, index+1) - if res_8: - for cp in rutf8.Utf8StringIterator(res_8): + if rettype == 'u' and r: + for cp in rutf8.Utf8StringIterator(r): + if cp < 0xD800: + _STORECHAR32(result, cp, byteorder) + else: + errorhandler('strict', public_encoding_name, + 'malformed unicode', + s, index, index+1) + elif r: + for ch in r: + cp = ord(ch) if cp < 0xD800: _STORECHAR32(result, cp, byteorder) else: @@ -1422,16 +1466,26 @@ index += 1 continue if not allow_surrogates and 0xD800 <= ch < 0xE000: - res_8, newindex = errorhandler( + r, newindex, rettype = errorhandler( errors, public_encoding_name, 'surrogates not allowed', s, index, index+1) - for ch in rutf8.Utf8StringIterator(res_8): - if ch < 0xD800: - _STORECHAR32(result, ch, byteorder) - else: - errorhandler( - 'strict', public_encoding_name, 'surrogates not allowed', - s, index, index+1) + if rettype == 'u': + for ch in rutf8.Utf8StringIterator(r): + if ch < 0xD800: + _STORECHAR32(result, ch, byteorder) + else: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, index, index+1) + else: + for ch in r: + cp = ord(ch) + if cp < 0xD800: + _STORECHAR32(result, cp, byteorder) + else: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, index, index+1) if index != newindex: # Should be uncommon index = newindex pos = rutf8._pos_at_index(s, newindex) @@ -1471,11 +1525,20 @@ ch = ord(s[pos]) pos += 1 if errorhandler: - res_8, newindex = errorhandler( + r, newindex, rettype = errorhandler( errors, public_encoding_name, 'malformed unicode', s, pos - 1, pos) - if res_8: - for cp in rutf8.Utf8StringIterator(res_8): + if rettype == 'u' and r: + for cp in rutf8.Utf8StringIterator(r): + if cp < 0xD800: + _STORECHAR32(result, cp, byteorder) + else: + errorhandler('strict', public_encoding_name, + 'malformed unicode', + s, pos-1, pos) + elif r: + for ch in r: + cp = ord(ch) if cp < 0xD800: _STORECHAR32(result, cp, byteorder) else: @@ -1489,16 +1552,26 @@ index += 1 continue if not allow_surrogates and 0xD800 <= ch < 0xE000: - res_8, newindex = errorhandler( + r, newindex, rettype = errorhandler( errors, public_encoding_name, 'surrogates not allowed', s, pos - 1, pos) - for ch in rutf8.Utf8StringIterator(res_8): - if ch < 0xD800: - _STORECHAR32(result, ch, byteorder) - else: - errorhandler( - 'strict', public_encoding_name, 'surrogates not allowed', - s, pos - 1, pos) + if rettype == 'u': + for ch in rutf8.Utf8StringIterator(res_8): + if ch < 0xD800: + _STORECHAR32(result, ch, byteorder) + else: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + else: + for ch in res_8: + cp = ord(ch) + if cp < 0xD800: + _STORECHAR32(result, cp, byteorder) + else: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) if index != newindex: # Should be uncommon index = newindex pos = rutf8._pos_at_index(s, newindex) @@ -1551,10 +1624,10 @@ pos = 0 while pos < size: if pos > size - unicode_bytes: - res, pos = errorhandler(errors, "unicode_internal", + r, pos, rettype = errorhandler(errors, "unicode_internal", "truncated input", s, pos, size) - result.append(res) + result.append(r) continue t = r_uint(0) h = 0 @@ -1562,10 +1635,10 @@ t += r_uint(ord(s[pos + j])) << (h*8) h += 1 if t > runicode.MAXUNICODE: - res, pos = errorhandler(errors, "unicode_internal", + r, pos, rettype = errorhandler(errors, "unicode_internal", "unichr(%d) not in range" % (t,), s, pos, pos + unicode_bytes) - result.append(res) + result.append(r) continue rutf8.unichr_as_utf8_append(result, intmask(t), allow_surrogates=True) pos += unicode_bytes @@ -1627,7 +1700,7 @@ c = mapping.get(ord(ch), ERROR_CHAR) if c == ERROR_CHAR: - r, pos = errorhandler(errors, "charmap", + r, pos, rettype = errorhandler(errors, "charmap", "character maps to ", s, pos, pos + 1) result.append(r) @@ -1659,10 +1732,10 @@ mapping.get(rutf8.codepoint_at_pos(s, pos), '') == ''): pos = rutf8.next_codepoint_pos(s, pos) index += 1 - res_8, newindex = errorhandler(errors, "charmap", + r, newindex, rettype = errorhandler(errors, "charmap", "character maps to ", s, startindex, index) - for cp2 in rutf8.Utf8StringIterator(res_8): + for cp2 in rutf8.Utf8StringIterator(r): ch2 = mapping.get(cp2, '') if not ch2: errorhandler( @@ -1727,7 +1800,7 @@ i += 1 end_index = i msg = "invalid decimal Unicode string" - r, pos = errorhandler( + r, pos, retype = errorhandler( errors, 'decimal', msg, s, start_index, end_index) for ch in rutf8.Utf8StringIterator(r): if unicodedb.isspace(ch): diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -72,8 +72,11 @@ raise OperationError(space.w_TypeError, space.newtext(msg)) w_replace, w_newpos = space.fixedview(w_res, 2) - if not (space.isinstance_w(w_replace, space.w_unicode) or - (not decode and space.isinstance_w(w_replace, space.w_bytes))): + if space.isinstance_w(w_replace, space.w_unicode): + rettype = 'u' + elif encode and space.isinstance_w(w_replace, space.w_bytes): + rettype = 'b' + else: if decode: msg = ("decoding error handler must return " "(str, int) tuple") @@ -94,7 +97,7 @@ raise oefmt(space.w_IndexError, "position %d from error handler out of bounds", newpos) - return space.utf8_w(w_replace), newpos + return space.utf8_w(w_replace), newpos, rettype return call_errorhandler def make_decode_errorhandler(self, space): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -826,11 +826,14 @@ repl = "\u00E9" s = "\u5678".encode("latin-1", "test.bad_handler") assert s == b'\xe9' + raises(UnicodeEncodeError, "\u5678".encode, "ascii", + "test.bad_handler") def test_lone_surrogates(self): encodings = ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-32-le', 'utf-32-be') for encoding in encodings: + print('encoding', encoding) raises(UnicodeEncodeError, u'\ud800'.encode, encoding) assert (u'[\udc80]'.encode(encoding, "backslashreplace") == '[\\udc80]'.encode(encoding)) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1200,14 +1200,16 @@ if errors is None or errors == 'strict': utf8 = space.utf8_w(w_object) if encoding is None or encoding == 'utf-8': - #if rutf8.has_surrogates(utf8): - # utf8 = rutf8.reencode_utf8_with_surrogates(utf8) + if rutf8.has_surrogates(utf8): + # slow path + return encode_text(space, w_object, encoding, errors) return space.newbytes(utf8) elif encoding == 'ascii': try: rutf8.check_ascii(utf8) except rutf8.CheckError as a: - eh = unicodehelper.encode_error_handler(space) + state = space.fromcache(CodecState) + eh = state.encode_error_handler eh(None, "ascii", "ordinal not in range(128)", utf8, a.pos, a.pos + 1) assert False, "always raises" From pypy.commits at gmail.com Sun Nov 18 22:36:43 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:43 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: convert bytes_w to unicode_w Message-ID: <5bf22fcb.1c69fb81.3950b.a80a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95340:f35ec4a5f0b1 Date: 2018-11-18 19:35 -0800 http://bitbucket.org/pypy/pypy/changeset/f35ec4a5f0b1/ Log: convert bytes_w to unicode_w diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -383,6 +383,7 @@ check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): w_obj = space.getattr(w_exc, space.newtext('object')) + w_obj = space.convert_arg_to_w_unicode(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) encoding = space.text_w(space.getattr(w_exc, space.newtext('encoding'))) @@ -467,6 +468,7 @@ check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): w_obj = space.getattr(w_exc, space.newtext('object')) + w_obj = space.convert_arg_to_w_unicode(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) From pypy.commits at gmail.com Sun Nov 18 22:36:45 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 19:36:45 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bf22fcd.1c69fb81.a9043.d465@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95341:72b0f6fca9f0 Date: 2018-11-18 19:35 -0800 http://bitbucket.org/pypy/pypy/changeset/72b0f6fca9f0/ Log: merge py3.5 into branch diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1073,6 +1073,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -42,7 +42,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -75,7 +75,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -138,11 +138,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -156,7 +156,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print("Called", self.num) -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print(sys.executable, self.tmpfile) - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -61,18 +61,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = io.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = io.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -539,11 +539,16 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if space.isinstance_w(w_obj, space.w_unicode): + if space.isinstance_w(w_obj, space.w_bytes): + s = space.bytes_w(w_obj) + if not s: + return space.newtext('') + elif space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") - s = space.charbuf_w(w_obj) + else: + s = space.buffer_w(w_obj, 0) return _pyunicode_decode(space, s, encoding, errors) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): From pypy.commits at gmail.com Mon Nov 19 00:14:11 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 21:14:11 -0800 (PST) Subject: [pypy-commit] pypy py3.5: use old buffer interface Message-ID: <5bf246a3.1c69fb81.d5cb.0a66@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95342:e1b0f8e6c29c Date: 2018-11-18 21:11 -0800 http://bitbucket.org/pypy/pypy/changeset/e1b0f8e6c29c/ Log: use old buffer interface diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -539,7 +539,7 @@ elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") else: - s = space.buffer_w(w_obj, 0) + s = space.charbuf_w(w_obj) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Mon Nov 19 00:14:12 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 21:14:12 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge py3.5 into branch Message-ID: <5bf246a4.1c69fb81.dc55.09fc@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95343:dd77cb64ae34 Date: 2018-11-18 21:12 -0800 http://bitbucket.org/pypy/pypy/changeset/dd77cb64ae34/ Log: merge py3.5 into branch diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -539,7 +539,7 @@ elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") else: - s = space.buffer_w(w_obj, 0) + s = space.charbuf_w(w_obj) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Mon Nov 19 00:16:42 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Nov 2018 21:16:42 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bf2473a.1c69fb81.3aa62.3d09@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95344:3e70370ebc0e Date: 2018-11-18 21:15 -0800 http://bitbucket.org/pypy/pypy/changeset/3e70370ebc0e/ Log: merge py3.5 into branch diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -548,7 +548,7 @@ elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific raise oefmt(space.w_TypeError, "decoding bytearray is not supported") else: - s = space.buffer_w(w_obj, 0) + s = space.charbuf_w(w_obj) return _pyunicode_decode(space, s, encoding, errors) From pypy.commits at gmail.com Tue Nov 20 03:24:46 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Nov 2018 00:24:46 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix logic, fix interface for _multibytecodec Message-ID: <5bf3c4ce.1c69fb81.5c201.1387@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95345:1b408df3f46d Date: 2018-11-19 23:57 -0800 http://bitbucket.org/pypy/pypy/changeset/1b408df3f46d/ Log: fix logic, fix interface for _multibytecodec diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -74,7 +74,7 @@ w_replace, w_newpos = space.fixedview(w_res, 2) if space.isinstance_w(w_replace, space.w_unicode): rettype = 'u' - elif encode and space.isinstance_w(w_replace, space.w_bytes): + elif not decode and space.isinstance_w(w_replace, space.w_bytes): rettype = 'b' else: if decode: diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -154,9 +154,9 @@ replace = UNICODE_REPLACEMENT_CHARACTER else: assert errorcb - replace, end = errorcb(errors, namecb, reason, + replace, end, rettype = errorcb(errors, namecb, reason, stringdata, start, end) - # 'replace' is RPython unicode here + # 'replace' is UTF8 encoded unicode, rettype is 'u' lgt = rutf8.get_utf8_length(replace) inbuf = rffi.utf82wcharp(replace, lgt) try: @@ -265,9 +265,13 @@ replace = "?" else: assert errorcb - replace, end = errorcb(errors, namecb, reason, + replace, end, rettype = errorcb(errors, namecb, reason, unicodedata, start, end) + if rettype == 'u': + codec = pypy_cjk_enc_getcodec(encodebuf) + replace = encode(codec, replace, end - start) + lgt = len(replace) with rffi.scoped_nonmovingbuffer(replace) as inbuf: - r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, len(replace), end) + r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, lgt, end) if r == MBERR_NOMEMORY: raise MemoryError From pypy.commits at gmail.com Tue Nov 20 03:24:48 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Nov 2018 00:24:48 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: if encode errorhandler returns bytes, do not check it Message-ID: <5bf3c4d0.1c69fb81.2b32f.55f9@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95346:6c473117814d Date: 2018-11-20 00:24 -0800 http://bitbucket.org/pypy/pypy/changeset/6c473117814d/ Log: if encode errorhandler returns bytes, do not check it diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -326,9 +326,9 @@ result.append(chr(cp)) else: for ch in res: - if ord(ch) > 0x80: - errorhandler("strict", 'ascii', msg, s, startindex, index) - raise RuntimeError('error handler should not have returned') + #if ord(ch) > 0x80: + # errorhandler("strict", 'ascii', msg, s, startindex, index) + # raise RuntimeError('error handler should not have returned') result.append(ch) pos = rutf8._pos_at_index(s, newindex) return result.build() From pypy.commits at gmail.com Tue Nov 20 14:00:29 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Nov 2018 11:00:29 -0800 (PST) Subject: [pypy-commit] pypy default: Compatibility with pytest 4.0 Message-ID: <5bf459cd.1c69fb81.88230.fd3a@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95348:d19ac6eec77b Date: 2018-11-20 18:59 +0000 http://bitbucket.org/pypy/pypy/changeset/d19ac6eec77b/ Log: Compatibility with pytest 4.0 diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,7 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + py.test.skip("not available on pypy", allow_module_level=True) cffi_dir = os.path.dirname(cffi_opcode.__file__) From pypy.commits at gmail.com Tue Nov 20 14:52:28 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Nov 2018 11:52:28 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: remove debug cruft Message-ID: <5bf465fc.1c69fb81.8ac19.c968@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95349:ffc6c900709e Date: 2018-11-20 11:51 -0800 http://bitbucket.org/pypy/pypy/changeset/ffc6c900709e/ Log: remove debug cruft diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1195,7 +1195,6 @@ if errors is None or errors == 'strict': utf8 = space.utf8_w(w_object) if encoding is None or encoding == 'utf-8': - print 'encode_object', utf8 == '\xed\xb0\x80', encoding, errors try: rutf8.check_utf8(utf8, False) except rutf8.CheckError as a: @@ -1203,7 +1202,6 @@ eh(None, "utf-8", "surrogates not allowed", utf8, a.pos, a.pos + 1) assert False, "always raises" - print 'no surrogate' return space.newbytes(utf8) elif encoding == 'ascii': try: From pypy.commits at gmail.com Tue Nov 20 15:34:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 20 Nov 2018 12:34:20 -0800 (PST) Subject: [pypy-commit] pypy py3.6: (Ryan Hileman, alcarithemad) fix problem with variable annotations and ternary Message-ID: <5bf46fcc.1c69fb81.9c59f.bcd1@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.6 Changeset: r95350:c761510af951 Date: 2018-11-20 21:33 +0100 http://bitbucket.org/pypy/pypy/changeset/c761510af951/ Log: (Ryan Hileman, alcarithemad) fix problem with variable annotations and ternary expressions fixes issue 2916 diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -791,7 +791,7 @@ annotation = self.handle_expr(second.get_child(1)) value_expr = None if second.num_children() == 4: - value_expr = self.handle_testlist(second.get_child(-1)) + value_expr = self.handle_expr(second.get_child(-1)) return ast.AnnAssign(target_expr, annotation, value_expr, simple, stmt.get_lineno(), stmt.get_column()) else: # Normal assignment. diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py --- a/pypy/interpreter/test/test_annotations.py +++ b/pypy/interpreter/test/test_annotations.py @@ -99,6 +99,14 @@ assert False ''' + def test_ternary_expression_bug(self): + """ + class C: + var: bool = True if False else False + assert var is False + assert C.__annotations__ == {"var": bool} + """ + def test_reassigned___annotations__(self): ''' class C: From pypy.commits at gmail.com Tue Nov 20 15:47:09 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Nov 2018 12:47:09 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5bf472cd.1c69fb81.bc0de.aeff@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95351:5811240c082f Date: 2018-11-20 20:25 +0000 http://bitbucket.org/pypy/pypy/changeset/5811240c082f/ Log: hg merge default diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,7 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + py.test.skip("not available on pypy", allow_module_level=True) cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - From pypy.commits at gmail.com Tue Nov 20 15:47:11 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Nov 2018 12:47:11 -0800 (PST) Subject: [pypy-commit] pypy apptest-file: hg merge default Message-ID: <5bf472cf.1c69fb81.dd734.39db@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r95352:57be5eeee2e5 Date: 2018-11-20 20:25 +0000 http://bitbucket.org/pypy/pypy/changeset/57be5eeee2e5/ Log: hg merge default diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,7 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + py.test.skip("not available on pypy", allow_module_level=True) cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -977,6 +977,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -43,7 +43,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -76,7 +76,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -139,11 +139,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -157,7 +157,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print "Called", self.num -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print sys.executable, self.tmpfile - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -46,18 +46,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -159,6 +159,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -171,14 +175,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -195,6 +198,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -239,8 +247,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -421,6 +421,8 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.wrap(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -366,10 +366,14 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): if not encoding: # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) + w_str = space.newbytes(s) w_encoding = space.newtext(rffi.charp2str(encoding)) if errors: w_errors = space.newbytes(rffi.charp2str(errors)) @@ -398,28 +402,12 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: + if space.isinstance_w(w_obj, space.w_unicode): raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) - else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + if space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") + s = space.bufferstr_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([CONST_STRING], PyObject) def PyUnicode_FromString(space, s): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -210,9 +210,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print type(st).__module__ + # print type(st).__module__ s = pickle.dumps(st) - print repr(s) + # print repr(s) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -302,7 +302,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except OSError as e: + except (IOError, OSError) as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" @@ -575,6 +575,12 @@ assert '\nOSError: [Errno 9]' in res else: assert res == 'test1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -188,8 +188,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -225,7 +225,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -243,11 +242,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -23,13 +23,11 @@ import glob from pypy.tool.release.smartstrip import smartstrip -if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") - USE_ZIPFILE_MODULE = sys.platform == 'win32' STDLIB_VER = "2.7" -from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, +from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, MissingDependenciesError, cffi_build_scripts) def ignore_patterns(*patterns): @@ -149,7 +147,7 @@ # XXX users will complain that they cannot compile capi (cpyext) # modules for windows, also embedding pypy (i.e. in cffi) # will fail. - # Has the lib moved, was translation not 'shared', or are + # Has the lib moved, was translation not 'shared', or are # there no exported functions in the dll so no import # library was created? if not options.no_tk: @@ -309,7 +307,7 @@ if __name__ == '__main__': import sys if sys.platform == 'win32': - # Try to avoid opeing a dialog box if one of the + # Try to avoid opeing a dialog box if one of the # subprocesses causes a system error import ctypes winapi = ctypes.windll.kernel32 diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -88,6 +88,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Tue Nov 20 15:47:14 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Nov 2018 12:47:14 -0800 (PST) Subject: [pypy-commit] pypy py3tests: hg merge py3.5 Message-ID: <5bf472d2.1c69fb81.35b71.505f@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r95353:5bc653be8f68 Date: 2018-11-20 20:26 +0000 http://bitbucket.org/pypy/pypy/changeset/5bc653be8f68/ Log: hg merge py3.5 diff --git a/extra_tests/__init__.py b/extra_tests/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/__init__.py b/extra_tests/cffi_tests/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/__init__.py rename to extra_tests/cffi_tests/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py b/extra_tests/cffi_tests/cffi0/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/__init__.py rename to extra_tests/cffi_tests/cffi0/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py rename to extra_tests/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -3,7 +3,7 @@ import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * SIZE_OF_INT = ctypes.sizeof(ctypes.c_int) SIZE_OF_LONG = ctypes.sizeof(ctypes.c_long) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py b/extra_tests/cffi_tests/cffi0/callback_in_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/callback_in_thread.py rename to extra_tests/cffi_tests/cffi0/callback_in_thread.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_module/snip_basic_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_1/snip_basic_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/distutils_package_2/snip_basic_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/infrastructure/snip_infrastructure/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_module/snip_setuptools_verify.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_1/snip_setuptools_verify1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/setup.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py b/extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py rename to extra_tests/cffi_tests/cffi0/snippets/setuptools_package_2/snip_setuptools_verify2/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py b/extra_tests/cffi_tests/cffi0/test_cdata.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_cdata.py rename to extra_tests/cffi_tests/cffi0/test_cdata.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/extra_tests/cffi_tests/cffi0/test_ctypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py rename to extra_tests/cffi_tests/cffi0/test_ctypes.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py +++ b/extra_tests/cffi_tests/cffi0/test_ctypes.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests +from extra_tests.cffi_tests.cffi0 import backend_tests from cffi.backend_ctypes import CTypesBackend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py rename to extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -1,8 +1,8 @@ # Generated by pypy/tool/import_cffi.py import py, sys, platform import pytest -from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib +from extra_tests.cffi_tests.support import u from cffi import FFI import _cffi_backend diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py rename to extra_tests/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -4,8 +4,8 @@ import math, os, sys import ctypes.util from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import FdWriteCapture from .backend_tests import needs_dlopen_none try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py b/extra_tests/cffi_tests/cffi0/test_model.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_model.py rename to extra_tests/cffi_tests/cffi0/test_model.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py rename to extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -1,9 +1,9 @@ # Generated by pypy/tool/import_cffi.py -import py, sys +import py, sys, os import subprocess, weakref from cffi import FFI from cffi.backend_ctypes import CTypesBackend -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.support import u SOURCE = """\ @@ -115,10 +115,9 @@ def setup_class(cls): cls.module = None - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) if sys.platform == 'win32': - import os # did we already build it? if cls.Backend is CTypesBackend: dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend @@ -149,10 +148,23 @@ os.rename(str(udir) + '\\testownlib.dll', dll_path) cls.module = dll_path else: + encoded = None + if cls.Backend is not CTypesBackend: + try: + unicode_name = u+'testownlibcaf\xe9' + encoded = unicode_name.encode(sys.getfilesystemencoding()) + if sys.version_info >= (3,): + encoded = str(unicode_name) + except UnicodeEncodeError: + pass + if encoded is None: + unicode_name = u+'testownlib' + encoded = str(unicode_name) subprocess.check_call( - 'cc testownlib.c -shared -fPIC -o testownlib.so', + "cc testownlib.c -shared -fPIC -o '%s.so'" % (encoded,), cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + cls.module = os.path.join(str(udir), unicode_name + (u+'.so')) + print(repr(cls.module)) def test_getting_errno(self): if self.module is None: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py rename to extra_tests/cffi_tests/cffi0/test_parsing.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py b/extra_tests/cffi_tests/cffi0/test_platform.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_platform.py rename to extra_tests/cffi_tests/cffi0/test_platform.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py b/extra_tests/cffi_tests/cffi0/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi0/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py rename to extra_tests/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2,7 +2,7 @@ import py, re import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * lib_m = ['m'] @@ -1408,7 +1408,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1418,7 +1418,7 @@ def test_relative_to(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py b/extra_tests/cffi_tests/cffi0/test_verify2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify2.py rename to extra_tests/cffi_tests/cffi0/test_verify2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/extra_tests/cffi_tests/cffi0/test_version.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py rename to extra_tests/cffi_tests/cffi0/test_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py b/extra_tests/cffi_tests/cffi0/test_vgen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen.py rename to extra_tests/cffi_tests/cffi0/test_vgen.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py b/extra_tests/cffi_tests/cffi0/test_vgen2.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_vgen2.py rename to extra_tests/cffi_tests/cffi0/test_vgen2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py b/extra_tests/cffi_tests/cffi0/test_zdistutils.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py rename to extra_tests/cffi_tests/cffi0/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zdistutils.py +++ b/extra_tests/cffi_tests/cffi0/test_zdistutils.py @@ -4,7 +4,7 @@ from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir class DistUtilsTest(object): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py rename to extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -1,7 +1,7 @@ # Generated by pypy/tool/import_cffi.py import py, os, sys, shutil import subprocess -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir if sys.platform == 'win32': py.test.skip('snippets do not run on win32') diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py b/extra_tests/cffi_tests/cffi1/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/__init__.py rename to extra_tests/cffi_tests/cffi1/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py b/extra_tests/cffi_tests/cffi1/test_cffi_binary.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_cffi_binary.py rename to extra_tests/cffi_tests/cffi1/test_cffi_binary.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/extra_tests/cffi_tests/cffi1/test_commontypes.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py rename to extra_tests/cffi_tests/cffi1/test_commontypes.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py b/extra_tests/cffi_tests/cffi1/test_dlopen.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py rename to extra_tests/cffi_tests/cffi1/test_dlopen.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen.py +++ b/extra_tests/cffi_tests/cffi1/test_dlopen.py @@ -2,7 +2,7 @@ import py from cffi import FFI, VerificationError, CDefError from cffi.recompiler import make_py_source -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir def test_simple(): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_dlopen_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_dlopen_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py rename to extra_tests/cffi_tests/cffi1/test_ffi_obj.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py rename to extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -3,8 +3,8 @@ import platform, imp import sys, os, ctypes import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import * from cffi.recompiler import recompile from cffi.cffi_opcode import PRIMITIVE_TO_INDEX diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py rename to extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,7 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + py.test.skip("not available on pypy", allow_module_level=True) cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/extra_tests/cffi_tests/cffi1/test_re_python.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py rename to extra_tests/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/extra_tests/cffi_tests/cffi1/test_re_python.py @@ -3,8 +3,8 @@ import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u def setup_module(mod): @@ -37,13 +37,22 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + + # test with a non-ascii char + ofn, oext = os.path.splitext(outputfilename) if sys.platform == "win32": - # test with a non-ascii char - outputfn1 = outputfilename - ofn, oext = os.path.splitext(outputfn1) - outputfilename = ofn + (u+'\u03be') + oext - #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) - os.rename(outputfn1, outputfilename) + unicode_name = ofn + (u+'\u03be') + oext + else: + unicode_name = ofn + (u+'\xe9') + oext + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None + if unicode_name is not None: + print(repr(outputfilename) + ' ==> ' + repr(unicode_name)) + os.rename(outputfilename, unicode_name) + outputfilename = unicode_name + mod.extmod = outputfilename mod.tmpdir = tmpdir # diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/extra_tests/cffi_tests/cffi1/test_realize_c_type.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py rename to extra_tests/cffi_tests/cffi1/test_realize_c_type.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py rename to extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -3,9 +3,9 @@ import sys, os, py from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.udir import udir -from pypy.module.test_lib_pypy.cffi_tests.support import u, long -from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture, StdErrCapture +from extra_tests.cffi_tests.udir import udir +from extra_tests.cffi_tests.support import u, long +from extra_tests.cffi_tests.support import FdWriteCapture, StdErrCapture try: import importlib diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py b/extra_tests/cffi_tests/cffi1/test_unicode_literals.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_unicode_literals.py rename to extra_tests/cffi_tests/cffi1/test_unicode_literals.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py rename to extra_tests/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -3,7 +3,7 @@ from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler -from pypy.module.test_lib_pypy.cffi_tests.support import * +from extra_tests.cffi_tests.support import * import _cffi_backend lib_m = ['m'] @@ -1377,7 +1377,7 @@ def test_tmpdir(): import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -1388,7 +1388,7 @@ def test_relative_to(): py.test.skip("not available") import tempfile, os - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpdir = tempfile.mkdtemp(dir=str(udir)) ffi = FFI() ffi.cdef("int foo(int);") @@ -2234,7 +2234,7 @@ def test_windows_dllimport_data(): if sys.platform != 'win32': py.test.skip("Windows only") - from pypy.module.test_lib_pypy.cffi_tests.udir import udir + from extra_tests.cffi_tests.udir import udir tmpfile = udir.join('dllimport_data.c') tmpfile.write('int my_value = 42;\n') ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/extra_tests/cffi_tests/cffi1/test_zdist.py rename from pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py rename to extra_tests/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/extra_tests/cffi_tests/cffi1/test_zdist.py @@ -2,7 +2,7 @@ import sys, os, py import subprocess import cffi -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir from shutil import rmtree from tempfile import mkdtemp diff --git a/pypy/module/test_lib_pypy/cffi_tests/conftest.py b/extra_tests/cffi_tests/conftest.py rename from pypy/module/test_lib_pypy/cffi_tests/conftest.py rename to extra_tests/cffi_tests/conftest.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py b/extra_tests/cffi_tests/embedding/__init__.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py rename to extra_tests/cffi_tests/embedding/__init__.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/extra_tests/cffi_tests/embedding/add1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c rename to extra_tests/cffi_tests/embedding/add1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py b/extra_tests/cffi_tests/embedding/add1.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py rename to extra_tests/cffi_tests/embedding/add1.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py +++ b/extra_tests/cffi_tests/embedding/add1.py @@ -12,7 +12,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/extra_tests/cffi_tests/embedding/add2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c rename to extra_tests/cffi_tests/embedding/add2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py b/extra_tests/cffi_tests/embedding/add2.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py rename to extra_tests/cffi_tests/embedding/add2.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py b/extra_tests/cffi_tests/embedding/add3.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py rename to extra_tests/cffi_tests/embedding/add3.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/extra_tests/cffi_tests/embedding/add_recursive-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c rename to extra_tests/cffi_tests/embedding/add_recursive-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py b/extra_tests/cffi_tests/embedding/add_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py rename to extra_tests/cffi_tests/embedding/add_recursive.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py b/extra_tests/cffi_tests/embedding/empty.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/empty.py rename to extra_tests/cffi_tests/embedding/empty.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/extra_tests/cffi_tests/embedding/initerror.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py rename to extra_tests/cffi_tests/embedding/initerror.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/extra_tests/cffi_tests/embedding/perf-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c rename to extra_tests/cffi_tests/embedding/perf-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py b/extra_tests/cffi_tests/embedding/perf.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py rename to extra_tests/cffi_tests/embedding/perf.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py rename to extra_tests/cffi_tests/embedding/test_basic.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -2,7 +2,7 @@ import py import sys, os, re import shutil, subprocess, time -from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from extra_tests.cffi_tests.udir import udir import cffi diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py rename to extra_tests/cffi_tests/embedding/test_performance.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -1,6 +1,6 @@ # Generated by pypy/tool/import_cffi.py import sys -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': import py diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py b/extra_tests/cffi_tests/embedding/test_recursive.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py rename to extra_tests/cffi_tests/embedding/test_recursive.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py +++ b/extra_tests/cffi_tests/embedding/test_recursive.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestRecursive(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py b/extra_tests/cffi_tests/embedding/test_thread.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py rename to extra_tests/cffi_tests/embedding/test_thread.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py +++ b/extra_tests/cffi_tests/embedding/test_thread.py @@ -1,12 +1,12 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThread(EmbeddingTests): def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py b/extra_tests/cffi_tests/embedding/test_tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py rename to extra_tests/cffi_tests/embedding/test_tlocal.py --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py +++ b/extra_tests/cffi_tests/embedding/test_tlocal.py @@ -1,5 +1,5 @@ # Generated by pypy/tool/import_cffi.py -from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import EmbeddingTests +from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests class TestThreadLocal(EmbeddingTests): diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/extra_tests/cffi_tests/embedding/thread-test.h rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h rename to extra_tests/cffi_tests/embedding/thread-test.h diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/extra_tests/cffi_tests/embedding/thread1-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c rename to extra_tests/cffi_tests/embedding/thread1-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/extra_tests/cffi_tests/embedding/thread2-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c rename to extra_tests/cffi_tests/embedding/thread2-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c rename to extra_tests/cffi_tests/embedding/thread3-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/extra_tests/cffi_tests/embedding/tlocal-test.c rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c rename to extra_tests/cffi_tests/embedding/tlocal-test.c diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py b/extra_tests/cffi_tests/embedding/tlocal.py rename from pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py rename to extra_tests/cffi_tests/embedding/tlocal.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/support.py b/extra_tests/cffi_tests/support.py rename from pypy/module/test_lib_pypy/cffi_tests/support.py rename to extra_tests/cffi_tests/support.py --- a/pypy/module/test_lib_pypy/cffi_tests/support.py +++ b/extra_tests/cffi_tests/support.py @@ -9,7 +9,7 @@ return eval('u'+repr(other).replace(r'\\u', r'\u') .replace(r'\\U', r'\U')) u = U() - long = long # for further "from pypy.module.test_lib_pypy.cffi_tests.support import long" + long = long # for further "from extra_tests.cffi_tests.support import long" assert u+'a\x00b' == eval(r"u'a\x00b'") assert u+'a\u1234b' == eval(r"u'a\u1234b'") assert u+'a\U00012345b' == eval(r"u'a\U00012345b'") diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py b/extra_tests/cffi_tests/test_egg_version.py rename from pypy/module/test_lib_pypy/cffi_tests/test_egg_version.py rename to extra_tests/cffi_tests/test_egg_version.py diff --git a/pypy/module/test_lib_pypy/cffi_tests/udir.py b/extra_tests/cffi_tests/udir.py rename from pypy/module/test_lib_pypy/cffi_tests/udir.py rename to extra_tests/cffi_tests/udir.py diff --git a/extra_tests/test_interpreter.py b/extra_tests/test_interpreter.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_interpreter.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import pytest + + at pytest.fixture +def testfile(tmpdir): + tmpfile = tmpdir.join('test_execution_context') + tmpfile.write(""" +from __future__ import print_function +import gc +class X(object): + def __del__(self): + print("Called", self.num) +def f(): + x1 = X(); x1.num = 1 + x2 = X(); x2.num = 2 + x1.next = x2 +f() +gc.collect() +gc.collect() +""") + return tmpfile + + +def test_del_not_blocked(testfile): + # test the behavior fixed in r71420: before, only one __del__ + # would be called + import os, sys + if sys.platform == "win32": + cmdformat = '"%s" "%s"' + else: + cmdformat = "'%s' '%s'" + g = os.popen(cmdformat % (sys.executable, testfile), 'r') + data = g.read() + g.close() + assert 'Called 1' in data + assert 'Called 2' in data diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1073,6 +1073,7 @@ " foo = True\n") + at py.test.mark.skipif('config.getoption("runappdirect")') class AppTestAppMain: def setup_class(self): # ---------------------------------------- diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -42,7 +42,7 @@ class Action1(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('one') - + class Action2(executioncontext.AsyncAction): def perform(self, ec, frame): events.append('two') @@ -75,7 +75,7 @@ class Action1(executioncontext.AsyncAction): _count = 0 - + def perform(self, ec, frame): events.append('one') if self._count == 0: @@ -138,11 +138,11 @@ def test_llprofile(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) - + space = self.space space.getexecutioncontext().setllprofile(profile_func, space.w_None) space.appexec([], """(): @@ -156,7 +156,7 @@ l = [] seen = [] space = self.space - + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) @@ -190,10 +190,10 @@ check_snippet('max(1, 2, **{})', 'builtin max') check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') check_snippet('abs(val=0)', 'builtin abs') - + def test_llprofile_c_exception(self): l = [] - + def profile_func(space, w_arg, frame, event, w_aarg): assert w_arg is space.w_None l.append(event) @@ -308,7 +308,7 @@ space = self.space w_res = space.appexec([], """(): l = [] - + def profile(*args): l.append(sys.exc_info()[0]) @@ -327,45 +327,6 @@ """) -class AppTestDelNotBlocked: - - def setup_method(self, meth): - if not self.runappdirect: - py.test.skip("test is meant for running with py.test -A") - from rpython.tool.udir import udir - tmpfile = udir.join('test_execution_context') - tmpfile.write(""" -import gc -class X(object): - def __del__(self): - print("Called", self.num) -def f(): - x1 = X(); x1.num = 1 - x2 = X(); x2.num = 2 - x1.next = x2 -f() -gc.collect() -gc.collect() -""") - self.tmpfile = str(tmpfile) - self.w_tmpfile = self.space.wrap(self.tmpfile) - - def test_del_not_blocked(self): - # test the behavior fixed in r71420: before, only one __del__ - # would be called - import os, sys - print(sys.executable, self.tmpfile) - if sys.platform == "win32": - cmdformat = '"%s" "%s"' - else: - cmdformat = "'%s' '%s'" - g = os.popen(cmdformat % (sys.executable, self.tmpfile), 'r') - data = g.read() - g.close() - assert 'Called 1' in data - assert 'Called 2' in data - - class AppTestProfile: def test_return(self): diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -1,4 +1,7 @@ import py, sys +from os.path import abspath, commonprefix, dirname + +THIS_DIR = dirname(__file__) @py.test.mark.tryfirst def pytest_runtest_setup(item): @@ -29,10 +32,11 @@ py.test.skip(infomsg) def pytest_ignore_collect(path, config): + path = str(path) if py.path.local.sysfind('genreflex') is None and config.option.runappdirect: - return True # "can't run dummy tests in -A" + return commonprefix([path, THIS_DIR]) == THIS_DIR if disabled: - return True + return commonprefix([path, THIS_DIR]) == THIS_DIR disabled = None diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -169,12 +169,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py --- a/pypy/module/_io/test/test_bytesio.py +++ b/pypy/module/_io/test/test_bytesio.py @@ -111,12 +111,12 @@ # exc = raises(TypeError, readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # b.close() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -164,12 +164,12 @@ # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not str") # exc = raises(TypeError, f.readinto, memoryview(b"hello")) msg = str(exc.value) - print(msg) + # print(msg) assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -61,18 +61,22 @@ except ImportError: skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. - del warnings.showwarning + saved = warnings.showwarning + try: + del warnings.showwarning - stderr = sys.stderr - try: - sys.stderr = io.StringIO() - inner('test message') - result = sys.stderr.getvalue() + stderr = sys.stderr + try: + sys.stderr = io.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result + warnings.showwarning = saved def test_filename_none(self): import _warnings diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -179,6 +179,10 @@ lltype.free(self._buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): + if self._buffer: + delta_memory_pressure = -self.allocated * self.itemsize + else: + delta_memory_pressure = 0 if size > 0: if size > self.allocated or size < self.allocated / 2: if overallocate: @@ -191,14 +195,13 @@ some = 0 self.allocated = size + some byte_size = self.allocated * self.itemsize + delta_memory_pressure += byte_size if zero: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True, zero=True) + rffi.CCHARP.TO, byte_size, flavor='raw', zero=True) else: new_buffer = lltype.malloc( - rffi.CCHARP.TO, byte_size, flavor='raw', - add_memory_pressure=True) + rffi.CCHARP.TO, byte_size, flavor='raw') copy_bytes = min(size, self.len) * self.itemsize rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer), rffi.cast(rffi.VOIDP, self._buffer), @@ -215,6 +218,11 @@ lltype.free(self._buffer, flavor='raw') self._buffer = new_buffer self.len = size + # adds the difference between the old and the new raw-malloced + # size. If setlen() is called a lot on the same array object, + # it is important to take into account the fact that we also do + # lltype.free() above. + rgc.add_memory_pressure(delta_memory_pressure) def _fromiterable(self, w_seq): # used by fromsequence(). @@ -259,8 +267,10 @@ return None oldbuffer = self._buffer self._buffer = lltype.malloc(rffi.CCHARP.TO, - (self.len - (j - i)) * self.itemsize, flavor='raw', - add_memory_pressure=True) + (self.len - (j - i)) * self.itemsize, flavor='raw') + # Issue #2913: don't pass add_memory_pressure here, otherwise + # memory pressure grows but actual raw memory usage doesn't---we + # are freeing the old buffer at the end of this function. if i: rffi.c_memcpy( rffi.cast(rffi.VOIDP, self._buffer), diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -677,13 +677,15 @@ def test_decode_null_encoding(self, space): null_charp = lltype.nullptr(rffi.CCHARP.TO) u_text = u'abcdefg' - s_text = space.str_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp)) + s_text = space.bytes_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp)) b_text = rffi.str2charp(s_text) assert space.unicode_w(PyUnicode_Decode( space, b_text, len(s_text), null_charp, null_charp)) == u_text with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) + assert space.unicode_w(PyUnicode_FromEncodedObject( + space, space.newbytes(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) def test_mbcs(self, space): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -486,11 +486,16 @@ in the unicode() built-in function. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" - if not encoding: - # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead - encoding = PyUnicode_GetDefaultEncoding(space) - w_str = space.newbytes(rffi.charpsize2str(s, size)) - w_encoding = space.newtext(rffi.charp2str(encoding)) + return _pyunicode_decode(space, rffi.charpsize2str(s, size), + encoding, errors) + +def _pyunicode_decode(space, s, encoding, errors): + if encoding: + w_encoding = space.newtext(rffi.charp2str(encoding)) + else: + # python 3.4 changed to this from defaultencoding + w_encoding = space.newtext('utf-8') + w_str = space.newbytes(s) if errors: w_errors = space.newtext(rffi.charp2str(errors)) else: @@ -525,28 +530,17 @@ All other objects, including Unicode objects, cause a TypeError to be set.""" - if not encoding: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - w_encoding = space.newtext(rffi.charp2str(encoding)) - if errors: - w_errors = space.newtext(rffi.charp2str(errors)) + if space.isinstance_w(w_obj, space.w_bytes): + s = space.bytes_w(w_obj) + if not s: + return space.newtext('') + elif space.isinstance_w(w_obj, space.w_unicode): + raise oefmt(space.w_TypeError, "decoding str is not supported") + elif space.isinstance_w(w_obj, space.w_bytearray): # Python 2.x specific + raise oefmt(space.w_TypeError, "decoding bytearray is not supported") else: - w_errors = None - - # - unicode is disallowed - # - raise TypeError for non-string types - if space.isinstance_w(w_obj, space.w_unicode): - w_meth = None - else: - try: - w_meth = space.getattr(w_obj, space.newtext('decode')) - except OperationError as e: - if not e.match(space, space.w_AttributeError): - raise - w_meth = None - if w_meth is None: - raise oefmt(space.w_TypeError, "decoding Unicode is not supported") - return space.call_function(w_meth, w_encoding, w_errors) + s = space.charbuf_w(w_obj) + return _pyunicode_decode(space, s, encoding, errors) @cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -227,9 +227,9 @@ def test_pickle(self): import pickle, os st = self.posix.stat(os.curdir) - print(type(st).__module__) + # print(type(st).__module__) s = pickle.dumps(st) - print(repr(s)) + # print(repr(s)) new = pickle.loads(s) assert new == st assert type(new) is type(st) @@ -573,6 +573,12 @@ res = fp.read() assert res == '1\n' + if sys.platform == "win32": + # using startfile in app_startfile creates global state + test_popen.dont_track_allocations = True + test_popen_with.dont_track_allocations = True + test_popen_child_fds.dont_track_allocations = True + if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): # nt specific diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -236,8 +236,8 @@ #endif #include #include +#include -RPY_EXPORTED char *_pypy_init_home(void) { HMODULE hModule = 0; @@ -273,7 +273,6 @@ #include #include -RPY_EXPORTED char *_pypy_init_home(void) { Dl_info info; @@ -291,11 +290,27 @@ } """ +_source_code += """ +inline +void _pypy_init_free(char *p) +{ + free(p); +} +""" + +if we_are_translated(): + post_include_bits = [] +else: + # for tests + post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);', + 'RPY_EXPORTED void _pypy_init_free(char*);', + ] + _eci = ExternalCompilationInfo(separate_module_sources=[_source_code], - post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);']) + post_include_bits=post_include_bits) _eci = _eci.merge(rdynload.eci) pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP, _nowrapper=True, compilation_info=_eci) -pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void, +pypy_init_free = rffi.llexternal("_pypy_init_free", [rffi.CCHARP], lltype.Void, _nowrapper=True, compilation_info=_eci) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -11,7 +11,7 @@ if ext == '.py': yield "# Generated by pypy/tool/import_cffi.py\n" for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + line = line.replace('from testing', 'from extra_tests.cffi_tests') yield line elif ext in ('.c', '.h'): yield "/* Generated by pypy/tool/import_cffi.py */\n" @@ -22,10 +22,10 @@ def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) - pypydir = py.path.local(__file__).join('..', '..') - cffi_dest = pypydir.join('..', 'lib_pypy', 'cffi') + rootdir = py.path.local(__file__).join('..', '..', '..') + cffi_dest = rootdir / 'lib_pypy' / 'cffi' cffi_dest.ensure(dir=1) - test_dest = pypydir.join('module', 'test_lib_pypy', 'cffi_tests') + test_dest = rootdir / 'extra_tests' / 'cffi_tests' test_dest.ensure(dir=1) for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -85,6 +85,9 @@ stdout, stderr = popen.communicate() if popen.wait() != 0: return None + if stdout[:5].lower() == 'error': + log.msg('Running "%s" errored: %s' %(vcvars, stdout.split()[0])) + return None except: return None From pypy.commits at gmail.com Tue Nov 20 15:47:17 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Nov 2018 12:47:17 -0800 (PST) Subject: [pypy-commit] pypy py3tests: hg merge apptest-file Message-ID: <5bf472d5.1c69fb81.584c.962e@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r95354:c2e1ab58d77d Date: 2018-11-20 20:46 +0000 http://bitbucket.org/pypy/pypy/changeset/c2e1ab58d77d/ Log: hg merge apptest-file diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -25,6 +25,8 @@ from rpython.translator.platform import host from rpython.translator.tool.cbuild import ExternalCompilationInfo +from .. import VERSION as TEST_VERSION + class AppTestC(object): """Populated below, hack hack hack.""" @@ -32,6 +34,15 @@ spaceconfig = dict(usemodules=('_cffi_backend', '_io', 'array')) def setup_class(cls): + if cls.runappdirect: + _cffi_backend = py.test.importorskip('_cffi_backend') + if _cffi_backend.__version__ != TEST_VERSION: + py.test.skip( + "These tests are for cffi version %s, this Python " + "has version %s installed" % + (TEST_VERSION, _cffi_backend.__version__)) + + testfuncs_w = [] keepalive_funcs = [] UniqueCache.for_testing = True diff --git a/pypy/module/_continuation/test/conftest.py b/pypy/module/_continuation/test/conftest.py --- a/pypy/module/_continuation/test/conftest.py +++ b/pypy/module/_continuation/test/conftest.py @@ -1,7 +1,7 @@ -import pytest import sys def pytest_configure(config): - if sys.platform.startswith('linux'): + if (not config.getoption('runappdirect') and + sys.platform.startswith('linux')): from rpython.rlib.rvmprof.cintf import configure_libbacktrace_linux configure_libbacktrace_linux() diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -41,6 +41,8 @@ disabled = None def pytest_configure(config): + if config.getoption('runappdirect'): + return if py.path.local.sysfind('genreflex') is None: import pypy.module._cppyy.capi.loadable_capi as lcapi try: diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -16,10 +16,9 @@ space = gettestobjspace(usemodules=['time']) space.getbuiltinmodule("time") -def pytest_ignore_collect(path, config): # ensure additional functions are registered import pypy.module.cpyext.test.test_cpyext - return False + def pytest_funcarg__api(request): return request.cls.api diff --git a/pypy/module/pypyjit/test/conftest.py b/pypy/module/pypyjit/test/conftest.py deleted file mode 100644 --- a/pypy/module/pypyjit/test/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -def pytest_addoption(parser): - group = parser.getgroup("pypyjit options") - group.addoption("--pypy", action="store", default=None, dest="pypy_c", - help="DEPRECATED: use this in test_pypy_c instead") -# XXX kill the corresponding section in the buildbot run, -# which (as far as I can tell) ignores that option entirely and does -# the same as the regular py.test. From pypy.commits at gmail.com Wed Nov 21 00:49:57 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 20 Nov 2018 21:49:57 -0800 (PST) Subject: [pypy-commit] cffi default: Add compatibility with pytest 4.0 without breaking older pytests Message-ID: <5bf4f205.1c69fb81.3453b.2b00@mx.google.com> Author: Armin Rigo Branch: Changeset: r3171:828c24a88bf6 Date: 2018-11-21 07:49 +0200 http://bitbucket.org/cffi/cffi/changeset/828c24a88bf6/ Log: Add compatibility with pytest 4.0 without breaking older pytests diff --git a/testing/cffi1/test_parse_c_type.py b/testing/cffi1/test_parse_c_type.py --- a/testing/cffi1/test_parse_c_type.py +++ b/testing/cffi1/test_parse_c_type.py @@ -3,7 +3,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) From pypy.commits at gmail.com Wed Nov 21 19:43:16 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 21 Nov 2018 16:43:16 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix import Message-ID: <5bf5fba4.1c69fb81.16357.41e4@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95355:773010593365 Date: 2018-11-22 00:42 +0000 http://bitbucket.org/pypy/pypy/changeset/773010593365/ Log: fix import diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py --- a/extra_tests/test_decimal.py +++ b/extra_tests/test_decimal.py @@ -4,7 +4,7 @@ import pickle import sys -from support import import_fresh_module +from .support import import_fresh_module C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) From pypy.commits at gmail.com Thu Nov 22 05:00:04 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 22 Nov 2018 02:00:04 -0800 (PST) Subject: [pypy-commit] pypy expose-gc-time: expose the time that the GC takes as a stat Message-ID: <5bf67e24.1c69fb81.b022.6c01@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: expose-gc-time Changeset: r95356:6759b7682f7f Date: 2018-11-22 10:59 +0100 http://bitbucket.org/pypy/pypy/changeset/6759b7682f7f/ Log: expose the time that the GC takes as a stat diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -192,6 +193,17 @@ # ____________________________________________________________ +def count_gc_time(meth): + def timing_meth(self, *args): + t1 = time.time() + res = meth(self, *args) + t2 = time.time() + self.total_gc_time += t2 - t1 + return res + timing_meth.func_name = meth.func_name + "_timing" + return timing_meth + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +386,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1640,6 +1653,7 @@ # ---------- # Nursery collection + @count_gc_time def _minor_collection(self): """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" @@ -2248,6 +2262,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step + @count_gc_time def major_collection_step(self, reserving_size=0): start = read_timestamp() debug_start("gc-collect-step") @@ -3000,6 +3015,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 @@ -3093,6 +3110,7 @@ return llmemory.NULL + @count_gc_time def rrc_invoke_callback(self): if self.rrc_enabled and self.rrc_dealloc_pending.non_empty(): self.rrc_dealloc_trigger_callback() diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object): From pypy.commits at gmail.com Thu Nov 22 11:35:41 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 22 Nov 2018 08:35:41 -0800 (PST) Subject: [pypy-commit] pypy expose-gc-time: expose GC time to applevel Message-ID: <5bf6dadd.1c69fb81.66f8.d4ce@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: expose-gc-time Changeset: r95357:526c66940c43 Date: 2018-11-22 17:35 +0100 http://bitbucket.org/pypy/pypy/changeset/526c66940c43/ Log: expose GC time to applevel diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) From pypy.commits at gmail.com Fri Nov 23 17:02:25 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 23 Nov 2018 14:02:25 -0800 (PST) Subject: [pypy-commit] pypy py3.6: link to proper lib on win32 Message-ID: <5bf878f1.1c69fb81.9e038.2ce2@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95358:e428d9b17460 Date: 2018-11-23 15:46 -0600 http://bitbucket.org/pypy/pypy/changeset/e428d9b17460/ Log: link to proper lib on win32 diff --git a/pypy/module/cpyext/include/pyconfig.h b/pypy/module/cpyext/include/pyconfig.h --- a/pypy/module/cpyext/include/pyconfig.h +++ b/pypy/module/cpyext/include/pyconfig.h @@ -38,9 +38,9 @@ */ # ifdef _DEBUG # error("debug first with cpython") -# pragma comment(lib,"python35.lib") +# pragma comment(lib,"python36.lib") # else -# pragma comment(lib,"python35.lib") +# pragma comment(lib,"python36.lib") # endif /* _DEBUG */ # define HAVE_COPYSIGN 1 # define copysign _copysign From pypy.commits at gmail.com Fri Nov 23 17:02:27 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 23 Nov 2018 14:02:27 -0800 (PST) Subject: [pypy-commit] pypy py3.6: export function from dll Message-ID: <5bf878f3.1c69fb81.d369e.f12d@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r95359:e999d9a3df93 Date: 2018-11-23 15:47 -0600 http://bitbucket.org/pypy/pypy/changeset/e999d9a3df93/ Log: export function from dll diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h --- a/pypy/module/cpyext/include/pyerrors.h +++ b/pypy/module/cpyext/include/pyerrors.h @@ -17,6 +17,7 @@ PyAPI_FUNC(PyObject *) PyErr_NewException(const char *name, PyObject *base, PyObject *dict); PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict); PyAPI_FUNC(PyObject *) PyErr_Format(PyObject *exception, const char *format, ...); +PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause(PyObject *exception, const char *format, ...); /* These APIs aren't really part of the error implementation, but often needed to format error messages; the native C lib APIs are From pypy.commits at gmail.com Sun Nov 25 02:36:28 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 24 Nov 2018 23:36:28 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/828c24a88bf6 Message-ID: <5bfa50fc.1c69fb81.d968c.6c53@mx.google.com> Author: Armin Rigo Branch: Changeset: r95360:d5b5c118b132 Date: 2018-11-25 09:35 +0200 http://bitbucket.org/pypy/pypy/changeset/d5b5c118b132/ Log: update to cffi/828c24a88bf6 diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,12 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy", allow_module_level=True) + try: + # pytest >= 4.0 + py.test.skip("not available on pypy", allow_module_level=True) + except TypeError: + # older pytest + py.test.skip("not available on pypy") cffi_dir = os.path.dirname(cffi_opcode.__file__) From pypy.commits at gmail.com Sun Nov 25 13:32:49 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:49 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix test Message-ID: <5bfaead1.1c69fb81.9b608.4513@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95361:fad3f16aa899 Date: 2018-11-20 22:36 -0800 http://bitbucket.org/pypy/pypy/changeset/fad3f16aa899/ Log: fix test diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -689,7 +689,7 @@ with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) - assert space.unicode_w(PyUnicode_FromEncodedObject( + assert space.text_w(PyUnicode_FromEncodedObject( space, space.newbytes(s_text), null_charp, None)) == u_text rffi.free_charp(b_text) From pypy.commits at gmail.com Sun Nov 25 13:32:51 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:51 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add more tests Message-ID: <5bfaead3.1c69fb81.63b97.2313@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95362:a9ecac2f678b Date: 2018-11-23 08:59 -0600 http://bitbucket.org/pypy/pypy/changeset/a9ecac2f678b/ Log: add more tests diff --git a/pypy/module/_multibytecodec/test/test_app_incremental.py b/pypy/module/_multibytecodec/test/test_app_incremental.py --- a/pypy/module/_multibytecodec/test/test_app_incremental.py +++ b/pypy/module/_multibytecodec/test/test_app_incremental.py @@ -1,5 +1,6 @@ +import os class AppTestClasses: - spaceconfig = dict(usemodules=['_multibytecodec']) + spaceconfig = dict(usemodules=['_multibytecodec', '_codecs', '_io']) def setup_class(cls): cls.w_IncrementalHzDecoder = cls.space.appexec([], """(): @@ -29,6 +30,7 @@ return IncrementalBig5hkscsEncoder """) + cls.w_myfile = cls.space.wrap(os.path.dirname(__file__)) def test_decode_hz(self): d = self.IncrementalHzDecoder() @@ -170,3 +172,27 @@ assert r == b'\x88f' r = e.encode('\u0304') assert r == b'\x88b' + + def test_incremental_big5hkscs(self): + import _codecs, _io + with open(self.myfile + '/big5hkscs.txt', 'rb') as fid: + uni_str = fid.read() + with open(self.myfile + '/big5hkscs-utf8.txt', 'rb') as fid: + utf8str = fid.read() + UTF8Reader = _codecs.lookup('utf-8').streamreader + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(_io.BytesIO(utf8str)) + ostream = _io.BytesIO() + encoder = self.IncrementalBig5hkscsEncoder() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + e = encoder.encode(data) + ostream.write(e) + assert ostream.getvalue() == uni_str diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py --- a/pypy/module/_multibytecodec/test/test_c_codecs.py +++ b/pypy/module/_multibytecodec/test/test_c_codecs.py @@ -21,10 +21,10 @@ def test_decode_hz(): # stateful c = getcodec("hz") - u = decode(c, "~{abc}") - assert u == u'\u5f95\u6cef'.encode('utf8') + utf8 = decode(c, "~{abc}") + assert utf8.decode('utf8') == u'\u5f95\u6cef' u = decode(c, "~{") - assert u == '' + assert u == u'' def test_decodeex_hz(): c = getcodec("hz") @@ -85,13 +85,13 @@ def test_decode_hz_ignore(): c = getcodec("hz") - u = decode(c, 'def~{}abc', 'ignore') - assert u == u'def\u5fcf'.encode('utf8') + utf8 = decode(c, 'def~{}abc', 'ignore') + assert utf8.decode('utf8') == u'def\u5f95' def test_decode_hz_replace(): c = getcodec("hz") - u = decode(c, 'def~{}abc', 'replace') - assert u == u'def\ufffd\u5fcf'.encode('utf8') + utf8 = decode(c, 'def~{}abc', 'replace') + assert utf8.decode('utf8') == u'def\ufffd\u5f95\ufffd' def test_encode_hz(): c = getcodec("hz") @@ -130,3 +130,4 @@ return u'\xc3'.encode('utf8'), endingpos s = encode(c, u'abc\u1234def'.encode('utf8'), 7, 'foo', errorhandler) assert '\xc3' in s + From pypy.commits at gmail.com Sun Nov 25 13:32:52 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:52 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix wrong unicode length Message-ID: <5bfaead4.1c69fb81.ca6ec.cf73@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95363:a325699736bc Date: 2018-11-23 08:59 -0600 http://bitbucket.org/pypy/pypy/changeset/a325699736bc/ Log: fix wrong unicode length diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -269,7 +269,8 @@ unicodedata, start, end) if rettype == 'u': codec = pypy_cjk_enc_getcodec(encodebuf) - replace = encode(codec, replace, end - start) + lgt = rutf8.check_utf8(replace, False) + replace = encode(codec, replace, lgt) lgt = len(replace) with rffi.scoped_nonmovingbuffer(replace) as inbuf: r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, lgt, end) From pypy.commits at gmail.com Sun Nov 25 13:32:54 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:54 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add missing test files Message-ID: <5bfaead6.1c69fb81.14e11.9b1e@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95364:2a5ae5b4e029 Date: 2018-11-23 09:26 -0600 http://bitbucket.org/pypy/pypy/changeset/2a5ae5b4e029/ Log: add missing test files diff --git a/pypy/module/_multibytecodec/test/big5hkscs-utf8.txt b/pypy/module/_multibytecodec/test/big5hkscs-utf8.txt new file mode 100644 --- /dev/null +++ b/pypy/module/_multibytecodec/test/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/pypy/module/_multibytecodec/test/big5hkscs.txt b/pypy/module/_multibytecodec/test/big5hkscs.txt new file mode 100644 --- /dev/null +++ b/pypy/module/_multibytecodec/test/big5hkscs.txt @@ -0,0 +1,2 @@ +�E�\�s�ڍ� +�f�b�� ���� diff --git a/pypy/module/_multibytecodec/test/test_multibtye_codecs.py b/pypy/module/_multibytecodec/test/test_multibtye_codecs.py new file mode 100644 --- /dev/null +++ b/pypy/module/_multibytecodec/test/test_multibtye_codecs.py @@ -0,0 +1,64 @@ +import os + +class AppTestPartialEvaluation: + spaceconfig = dict(usemodules=['_multibytecodec', '_codecs']) + + def setup_class(cls): + cls.w_myfile = cls.space.wrap(os.path.dirname(__file__)) + + def test_callback_None_index(self): + import _multibytecodec, _codecs + codec = _multibytecodec.__getcodec('cp932') + def myreplace(exc): + return ('x', None) + _codecs.register_error("test.cjktest", myreplace) + raises(TypeError, codec.encode, '\udeee', 'test.cjktest') + + def test_callback_backward_index(self): + import _multibytecodec, _codecs + codec = _multibytecodec.__getcodec('cp932') + def myreplace(exc): + if myreplace.limit > 0: + myreplace.limit -= 1 + return ('REPLACED', 0) + else: + return ('TERMINAL', exc.end) + myreplace.limit = 3 + _codecs.register_error("test.cjktest", myreplace) + assert (codec.encode('abcd' + '\udeee' + 'efgh', 'test.cjktest') == + (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9)) + + def test_callback_forward_index(self): + import _multibytecodec, _codecs + codec = _multibytecodec.__getcodec('cp932') + def myreplace(exc): + return ('REPLACED', exc.end + 2) + _codecs.register_error("test.cjktest", myreplace) + assert (codec.encode('abcd' + '\udeee' + 'efgh', 'test.cjktest') == + (b'abcdREPLACEDgh', 9)) + + def _test_incrementalencoder(self): + import _multibytecodec, _codecs, _io + with open(self.myfile + '/shift_jis.txt', 'rb') as fid: + uni_str = fid.read() + with open(self.myfile + '/shift_jis-utf8.txt', 'rb') as fid: + utf8str = fid.read() + UTF8Reader = _codecs.lookup('utf-8').streamreader + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(_io.BytesIO(utf8str)) + ostream = _io.BytesIO() + codec = _multibytecodec.__getcodec('cp932') + print(dir(codec)) + encoder = codec.incrementalencoder() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + e = encoder.encode(data) + ostream.write(e) + assert ostream.getvalue() == uni_str From pypy.commits at gmail.com Sun Nov 25 13:32:56 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:56 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: return consumed, not unicode lgt Message-ID: <5bfaead8.1c69fb81.c9fc.a014@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95365:5ff7a09cd178 Date: 2018-11-24 00:02 -0600 http://bitbucket.org/pypy/pypy/changeset/5ff7a09cd178/ Log: return consumed, not unicode lgt diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -754,7 +754,7 @@ res, lgt, pos = unicodehelper.str_decode_utf8(string, errors, final, state.decode_error_handler) return space.newtuple([space.newutf8(res, lgt), - space.newint(lgt)]) + space.newint(pos)]) else: return space.newtuple([space.newutf8(string, lgt), space.newint(len(string))]) From pypy.commits at gmail.com Sun Nov 25 13:32:58 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:32:58 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: improve and add a test Message-ID: <5bfaeada.1c69fb81.dd734.a054@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95366:2d33f43487c5 Date: 2018-11-25 12:30 -0600 http://bitbucket.org/pypy/pypy/changeset/2d33f43487c5/ Log: improve and add a test diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py --- a/pypy/module/_locale/test/test_locale.py +++ b/pypy/module/_locale/test/test_locale.py @@ -20,6 +20,7 @@ # check whether used locales are installed, otherwise the tests will # fail current = _locale.setlocale(_locale.LC_ALL) + cls.oldlocale = current try: try: # some systems are only UTF-8 oriented @@ -45,6 +46,12 @@ finally: _locale.setlocale(_locale.LC_ALL, current) + def teardown_class(cls): + import _locale + _locale.setlocale(_locale.LC_ALL, cls.oldlocale) + + + def test_import(self): import _locale assert _locale @@ -299,3 +306,36 @@ assert lang is None or isinstance(lang, str) assert encoding.startswith('cp') + def test_lc_numeric_basic(self): + from _locale import (setlocale, nl_langinfo, Error, LC_NUMERIC, + LC_CTYPE, RADIXCHAR, THOUSEP, localeconv) + # Test nl_langinfo against localeconv + candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT', + 'et_EE', 'es_PY', 'no_NO', 'nl_NL', 'lv_LV', 'el_GR', 'be_BY', 'fr_BE', + 'ro_RO', 'ru_UA', 'ru_RU', 'es_VE', 'ca_ES', 'se_NO', 'es_EC', 'id_ID', + 'ka_GE', 'es_CL', 'wa_BE', 'hu_HU', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR', + 'es_ES', 'oc_FR', 'gl_ES', 'bg_BG', 'is_IS', 'mk_MK', 'de_AT', 'pt_BR', + 'da_DK', 'nn_NO', 'cs_CZ', 'de_LU', 'es_BO', 'sq_AL', 'sk_SK', 'fr_CH', + 'de_DE', 'sr_YU', 'br_FR', 'nl_BE', 'sv_FI', 'pl_PL', 'fr_CA', 'fo_FO', + 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', + 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', + 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15 at euro', + 'ru_RU.KOI8-R', 'ko_KR.eucKR'] + + tested = False + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + setlocale(LC_CTYPE, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + nl_radixchar = nl_langinfo(li) + li_radixchar = localeconv()[lc] + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + assert nl_radixchar == li_radixchar, ("nl_langinfo != localeconv " + "(set to %s, using %s)" % ( loc, set_locale)) From pypy.commits at gmail.com Sun Nov 25 13:33:00 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Nov 2018 10:33:00 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: UnicodeListStrategy can hold utf8, not just ascii Message-ID: <5bfaeadc.1c69fb81.135dc.b4b1@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95367:a841b6df8847 Date: 2018-11-25 12:31 -0600 http://bitbucket.org/pypy/pypy/changeset/a841b6df8847/ Log: UnicodeListStrategy can hold utf8, not just ascii diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -98,7 +98,6 @@ return self._utf8 def listview_utf8(self): - assert self.is_ascii() return _create_list_from_unicode(self._utf8) def ord(self, space): From pypy.commits at gmail.com Mon Nov 26 15:17:32 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Nov 2018 12:17:32 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix import on python3 Message-ID: <5bfc54dc.1c69fb81.e0ef6.d8ce@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95368:11d6f6bd6dc8 Date: 2018-11-26 12:16 -0800 http://bitbucket.org/pypy/pypy/changeset/11d6f6bd6dc8/ Log: fix import on python3 diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1542,7 +1542,7 @@ def _verify(ffi, module_name, preamble, *args, **kwds): # FOR TESTS ONLY - from testing.udir import udir + from .testing.udir import udir import imp assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) From pypy.commits at gmail.com Tue Nov 27 14:50:38 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 27 Nov 2018 11:50:38 -0800 (PST) Subject: [pypy-commit] cffi default: Silence bogus warning Message-ID: <5bfda00e.1c69fb81.4f64c.1c00@mx.google.com> Author: Armin Rigo Branch: Changeset: r3172:355a99f0c4cc Date: 2018-11-27 21:49 +0200 http://bitbucket.org/cffi/cffi/changeset/355a99f0c4cc/ Log: Silence bogus warning diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -156,7 +156,6 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) - _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) @@ -173,6 +172,9 @@ # Replace `extern "Python"` with start/end markers csource = _preprocess_extern_python(csource) # + # Now there should not be any string literal left; warn if we get one + _warn_for_string_literal(csource) + # # Replace "[...]" with "[__dotdotdotarray__]" csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) # diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -1538,15 +1538,18 @@ assert (pt.x, pt.y) == (99*500*999, -99*500*999) def test_extern_python_1(): + import warnings ffi = FFI() - ffi.cdef(""" + with warnings.catch_warnings(record=True) as log: + ffi.cdef(""" extern "Python" { int bar(int, int); void baz(int, int); int bok(void); void boz(void); } - """) + """) + assert len(log) == 0, "got a warning: %r" % (log,) lib = verify(ffi, 'test_extern_python_1', """ static void baz(int, int); /* forward */ """) From pypy.commits at gmail.com Tue Nov 27 21:02:37 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Nov 2018 18:02:37 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix allowing surrogates in PyUnicodeObject Message-ID: <5bfdf73d.1c69fb81.bc97e.32a9@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95370:33f7275c06e2 Date: 2018-11-25 18:52 -0600 http://bitbucket.org/pypy/pypy/changeset/33f7275c06e2/ Log: test, fix allowing surrogates in PyUnicodeObject diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -213,3 +213,25 @@ ''', PY_SSIZE_T_CLEAN=True) assert type(charbuf(b'12345')) is bytes assert charbuf(b'12345') == b'12345' + + def test_getargs_keywords(self): + # taken from lib-python/3/test_getargs2.py + module = self.import_extension('foo', [ + ("getargs_keywords", "METH_KEYWORDS | METH_VARARGS", + ''' + static char *keywords[] = {"arg1","arg2","arg3","arg4","arg5", NULL}; + static char *fmt="(ii)i|(i(ii))(iii)i"; + int int_args[10]={-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, fmt, keywords, + &int_args[0], &int_args[1], &int_args[2], &int_args[3], + &int_args[4], &int_args[5], &int_args[6], &int_args[7], + &int_args[8], &int_args[9])) + return NULL; + return Py_BuildValue("iiiiiiiiii", + int_args[0], int_args[1], int_args[2], int_args[3], int_args[4], + int_args[5], int_args[6], int_args[7], int_args[8], int_args[9] + ); + ''')]) + raises(TypeError, module.getargs_keywords, (1,2), 3, (4,(5,6)), (7,8,9), **{'\uDC80': 10}) + diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -83,9 +83,10 @@ be modified after this call. """ s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj)) + s_utf8 = runicode.unicode_encode_utf_8(s, len(s), 'strict', + allow_surrogates=True) w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type)) w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type) - s_utf8 = runicode.unicode_encode_utf_8(s, len(s), 'strict') w_obj.__init__(s_utf8, len(s)) track_reference(space, py_obj, w_obj) return w_obj From pypy.commits at gmail.com Tue Nov 27 21:02:39 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Nov 2018 18:02:39 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bfdf73f.1c69fb81.68e9e.0b4e@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95371:c700c69056ba Date: 2018-11-26 22:23 -0800 http://bitbucket.org/pypy/pypy/changeset/c700c69056ba/ Log: merge py3.5 into branch diff --git a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py --- a/extra_tests/cffi_tests/cffi1/test_parse_c_type.py +++ b/extra_tests/cffi_tests/cffi1/test_parse_c_type.py @@ -4,7 +4,7 @@ from cffi import cffi_opcode if '__pypy__' in sys.builtin_module_names: - py.test.skip("not available on pypy") + py.test.skip("not available on pypy", allow_module_level=True) cffi_dir = os.path.dirname(cffi_opcode.__file__) diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py --- a/extra_tests/test_decimal.py +++ b/extra_tests/test_decimal.py @@ -4,7 +4,7 @@ import pickle import sys -from support import import_fresh_module +from .support import import_fresh_module C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1542,7 +1542,7 @@ def _verify(ffi, module_name, preamble, *args, **kwds): # FOR TESTS ONLY - from testing.udir import udir + from .testing.udir import udir import imp assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) diff --git a/pypy/module/_rawffi/alt/test/test_ffitype.py b/pypy/module/_rawffi/alt/test/test_ffitype.py --- a/pypy/module/_rawffi/alt/test/test_ffitype.py +++ b/pypy/module/_rawffi/alt/test/test_ffitype.py @@ -1,6 +1,5 @@ -from pypy.module._rawffi.alt.test.test_funcptr import BaseAppTestFFI - -class AppTestFFIType(BaseAppTestFFI): +class AppTestFFIType(object): + spaceconfig = dict(usemodules=('_rawffi',)) def test_simple_types(self): from _rawffi.alt import types @@ -8,7 +7,7 @@ assert str(types.uint) == "" assert types.sint.name == 'sint' assert types.uint.name == 'uint' - + def test_sizeof(self): from _rawffi.alt import types assert types.sbyte.sizeof() == 1 @@ -36,4 +35,3 @@ assert x is types.char_p x = types.Pointer(types.unichar) assert x is types.unichar_p - From pypy.commits at gmail.com Tue Nov 27 21:02:41 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Nov 2018 18:02:41 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: add two forms of the same test, AppTest fails Message-ID: <5bfdf741.1c69fb81.d18f2.46d7@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95372:1bd2b4a25f02 Date: 2018-11-27 18:01 -0800 http://bitbucket.org/pypy/pypy/changeset/1bd2b4a25f02/ Log: add two forms of the same test, AppTest fails diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -25,6 +25,8 @@ assert self.space.eq_w(self.space.len(w_list), w(1)) w_list = W_ListObject(self.space, [w(5), w(3), w(99)]*111) assert self.space.eq_w(self.space.len(w_list), w(333)) + w_list = W_ListObject(self.space, [w(u'\u2039')]) + assert self.space.eq_w(self.space.len(w_list), w(1)) def test_getitem(self): w = self.space.wrap @@ -1397,6 +1399,10 @@ l3 = [s] assert l3[0].encode("ascii", "replace") == b"???" + s = u"\u2039" + l1 = list(s) + assert len(l1) == 1 + def test_list_from_set(self): l = ['a'] l.__init__(set('b')) From pypy.commits at gmail.com Tue Nov 27 22:42:47 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 27 Nov 2018 19:42:47 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Merge test_exception_extra.py into test_exc.py Message-ID: <5bfe0eb7.1c69fb81.b9f2f.98a5@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95373:df2db0bd9619 Date: 2018-11-28 03:16 +0000 http://bitbucket.org/pypy/pypy/changeset/df2db0bd9619/ Log: Merge test_exception_extra.py into test_exc.py diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py --- a/pypy/module/exceptions/test/test_exc.py +++ b/pypy/module/exceptions/test/test_exc.py @@ -4,6 +4,10 @@ class AppTestExc(object): spaceconfig = dict(usemodules=('exceptions',)) + def test_import(self): + import exceptions + assert exceptions.SyntaxError is SyntaxError + def test_baseexc(self): from exceptions import BaseException @@ -117,6 +121,8 @@ assert ee.strerror == "x" assert ee.filename == "y" assert EnvironmentError(3, "x").filename is None + e = EnvironmentError(1, "hello", "world") + assert str(e) == "[Errno 1] hello: 'world'" def test_windows_error(self): try: @@ -145,6 +151,7 @@ def test_system_exit(self): from exceptions import SystemExit + assert issubclass(SystemExit, BaseException) assert SystemExit().code is None assert SystemExit("x").code == "x" assert SystemExit(1, 2).code == (1, 2) diff --git a/pypy/module/test_lib_pypy/test_exception_extra.py b/pypy/module/test_lib_pypy/test_exception_extra.py deleted file mode 100644 --- a/pypy/module/test_lib_pypy/test_exception_extra.py +++ /dev/null @@ -1,21 +0,0 @@ -def app_test_environmenterror_repr(): - import exceptions as ex - e = ex.EnvironmentError("hello") - assert str(e) == "hello" - e = ex.EnvironmentError(1, "hello") - assert str(e) == "[Errno 1] hello" - e = ex.EnvironmentError(1, "hello", "world") - assert str(e) == "[Errno 1] hello: 'world'" - -def app_test_import(): - import exceptions - assert exceptions.SyntaxError is SyntaxError - -def app_test_baseexception(): - assert issubclass(Exception, BaseException) - -def app_test_systemexit(): - assert issubclass(SystemExit, BaseException) - -def app_test_keyboardinterrupt(): - assert issubclass(KeyboardInterrupt, BaseException) From pypy.commits at gmail.com Tue Nov 27 22:42:49 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 27 Nov 2018 19:42:49 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_string_extra.py to extra_tests/ Message-ID: <5bfe0eb9.1c69fb81.703b9.9cd4@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95374:d11ea562df99 Date: 2018-11-28 03:38 +0000 http://bitbucket.org/pypy/pypy/changeset/d11ea562df99/ Log: Move test_string_extra.py to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_string_extra.py b/extra_tests/test_string.py rename from pypy/module/test_lib_pypy/test_string_extra.py rename to extra_tests/test_string.py --- a/pypy/module/test_lib_pypy/test_string_extra.py +++ b/extra_tests/test_string.py @@ -1,8 +1,10 @@ + """ Test module for functions in string.py """ +import pytest -def app_test_maketrans(): +def test_maketrans(): import string assert string.maketrans('', '') == ( '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12' @@ -40,4 +42,5 @@ '\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2' '\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3' '\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') - raises(ValueError, string.maketrans, 'aa', '') + with pytest.raises(ValueError): + string.maketrans('aa', '') From pypy.commits at gmail.com Wed Nov 28 01:10:30 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Nov 2018 22:10:30 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: fix list(str) Message-ID: <5bfe3156.1c69fb81.93e4d.6502@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95375:4af05f233eaa Date: 2018-11-27 22:09 -0800 http://bitbucket.org/pypy/pypy/changeset/4af05f233eaa/ Log: fix list(str) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -3,7 +3,7 @@ from rpython.rlib.objectmodel import ( compute_hash, compute_unique_id, import_from_mixin, always_inline, enforceargs, newlist_hint, specialize, we_are_translated) -from rpython.rlib.rarithmetic import ovfcheck +from rpython.rlib.rarithmetic import ovfcheck, r_uint from rpython.rlib.rstring import ( StringBuilder, split, rsplit, UnicodeBuilder, replace_count, startswith, endswith) @@ -1861,7 +1861,7 @@ def _create_list_from_unicode(value): # need this helper function to allow the jit to look inside and inline # listview_unicode - return [s for s in value] + return [rutf8.unichr_as_utf8(r_uint(s), True) for s in rutf8.Utf8StringIterator(value)] W_UnicodeObject.EMPTY = W_UnicodeObject('', 0) From pypy.commits at gmail.com Wed Nov 28 09:28:34 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Nov 2018 06:28:34 -0800 (PST) Subject: [pypy-commit] pypy expose-gc-time: switch to use time.time() instead of read_timestamp for the GC time measuring Message-ID: <5bfea612.1c69fb81.631de.d5d5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: expose-gc-time Changeset: r95376:24e0860e66ac Date: 2018-11-28 15:27 +0100 http://bitbucket.org/pypy/pypy/changeset/24e0860e66ac/ Log: switch to use time.time() instead of read_timestamp for the GC time measuring in hooks and stats - this gives much more directly useful results - it's plenty fast enough: on my machine one time.time call takes ~40ns, read_timestamp about ~10ns, while a minor gc step takes ~9000ns diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -74,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -193,15 +192,6 @@ # ____________________________________________________________ -def count_gc_time(meth): - def timing_meth(self, *args): - t1 = time.time() - res = meth(self, *args) - t2 = time.time() - self.total_gc_time += t2 - t1 - return res - timing_meth.func_name = meth.func_name + "_timing" - return timing_meth class IncrementalMiniMarkGC(MovingGCBase): @@ -1653,12 +1643,11 @@ # ---------- # Nursery collection - @count_gc_time def _minor_collection(self): """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1857,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2262,9 +2252,8 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step - @count_gc_time def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2508,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3110,7 +3100,6 @@ return llmemory.NULL - @count_gc_time def rrc_invoke_callback(self): if self.rrc_enabled and self.rrc_dealloc_pending.non_empty(): self.rrc_dealloc_trigger_callback() diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) From pypy.commits at gmail.com Wed Nov 28 15:46:42 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Nov 2018 12:46:42 -0800 (PST) Subject: [pypy-commit] pypy expose-gc-time: add whatsnew item Message-ID: <5bfefeb2.1c69fb81.628f8.a090@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: expose-gc-time Changeset: r95377:b3db7a850036 Date: 2018-11-28 21:44 +0100 http://bitbucket.org/pypy/pypy/changeset/b3db7a850036/ Log: add whatsnew item diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -51,3 +51,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). From pypy.commits at gmail.com Wed Nov 28 16:31:43 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Nov 2018 13:31:43 -0800 (PST) Subject: [pypy-commit] pypy default: Modernise test_sqlite3.py and test only the host's _sqlite3 Message-ID: <5bff093f.1c69fb81.d8114.aa27@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95378:10acf469230d Date: 2018-11-28 21:30 +0000 http://bitbucket.org/pypy/pypy/changeset/10acf469230d/ Log: Modernise test_sqlite3.py and test only the host's _sqlite3 diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in e.value.message + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in excinfo.value.message - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in e.value.message +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in excinfo.value.message - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success From pypy.commits at gmail.com Thu Nov 29 00:58:01 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Nov 2018 21:58:01 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: hg merge default Message-ID: <5bff7fe9.1c69fb81.f0482.58fb@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95379:b58f67cf5f85 Date: 2018-11-29 05:09 +0000 http://bitbucket.org/pypy/pypy/changeset/b58f67cf5f85/ Log: hg merge default diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -4,8 +4,10 @@ from errno import EINVAL, EPERM import _structseq, os -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f class error(Exception): @@ -35,7 +37,7 @@ ru_oublock = _structseq.structseqfield(10, "block output operations") ru_msgsnd = _structseq.structseqfield(11, "IPC messages sent") ru_msgrcv = _structseq.structseqfield(12, "IPC messages received") - ru_nsignals = _structseq.structseqfield(13,"signals received") + ru_nsignals = _structseq.structseqfield(13, "signals received") ru_nvcsw = _structseq.structseqfield(14, "voluntary context switches") ru_nivcsw = _structseq.structseqfield(15, "involuntary context switches") @@ -57,7 +59,7 @@ ru.ru_nsignals, ru.ru_nvcsw, ru.ru_nivcsw, - )) + )) @builtinify def getrusage(who): diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/pypy/module/test_lib_pypy/test_sqlite3.py @@ -5,327 +5,321 @@ import pytest import sys +_sqlite3 = pytest.importorskip('_sqlite3') -def pytest_funcarg__con(request): +pypy_only = pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, + reason="PyPy-only test") + + + at pytest.yield_fixture +def con(): con = _sqlite3.connect(':memory:') - request.addfinalizer(lambda: con.close()) - return con + yield con + con.close() -class BaseTestSQLite: - def test_list_ddl(self, con): - """From issue996. Mostly just looking for lack of exceptions.""" - cursor = con.cursor() - cursor.execute('CREATE TABLE foo (bar INTEGER)') - result = list(cursor) - assert result == [] - cursor.execute('INSERT INTO foo (bar) VALUES (42)') - result = list(cursor) - assert result == [] - cursor.execute('SELECT * FROM foo') - result = list(cursor) - assert result == [(42,)] +def test_list_ddl(con): + """From issue996. Mostly just looking for lack of exceptions.""" + cursor = con.cursor() + cursor.execute('CREATE TABLE foo (bar INTEGER)') + result = list(cursor) + assert result == [] + cursor.execute('INSERT INTO foo (bar) VALUES (42)') + result = list(cursor) + assert result == [] + cursor.execute('SELECT * FROM foo') + result = list(cursor) + assert result == [(42,)] - def test_connect_takes_same_positional_args_as_Connection(self, con): - if not hasattr(_sqlite3, '_ffi'): - pytest.skip("only works for lib_pypy _sqlite3") - from inspect import getargspec - clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self - conargs = getargspec(_sqlite3.connect).args - assert clsargs == conargs + at pypy_only +def test_connect_takes_same_positional_args_as_Connection(con): + from inspect import getargspec + clsargs = getargspec(_sqlite3.Connection.__init__).args[1:] # ignore self + conargs = getargspec(_sqlite3.connect).args + assert clsargs == conargs - def test_total_changes_after_close(self, con): - con.close() - pytest.raises(_sqlite3.ProgrammingError, "con.total_changes") +def test_total_changes_after_close(con): + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + con.total_changes - def test_connection_check_init(self): - class Connection(_sqlite3.Connection): - def __init__(self, name): - pass +def test_connection_check_init(): + class Connection(_sqlite3.Connection): + def __init__(self, name): + pass - con = Connection(":memory:") - e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()") - assert '__init__' in e.value.message + con = Connection(":memory:") + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + con.cursor() + assert '__init__' in excinfo.value.message - def test_cursor_check_init(self, con): - class Cursor(_sqlite3.Cursor): - def __init__(self, name): - pass - cur = Cursor(con) - e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')") - assert '__init__' in e.value.message +def test_cursor_check_init(con): + class Cursor(_sqlite3.Cursor): + def __init__(self, name): + pass - def test_connection_after_close(self, con): - pytest.raises(TypeError, "con()") - con.close() - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "con()") + cur = Cursor(con) + with pytest.raises(_sqlite3.ProgrammingError) as excinfo: + cur.execute('select 1') + assert '__init__' in excinfo.value.message - def test_cursor_iter(self, con): +def test_connection_after_close(con): + with pytest.raises(TypeError): + con() + con.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + con() + +def test_cursor_iter(con): + cur = con.cursor() + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + next(cur) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + con.commit() + next(cur) + with pytest.raises(StopIteration): + next(cur) + + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany('select 1', []) + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('create table test(ing)') + with pytest.raises(StopIteration): + next(cur) + + cur.execute('select 1') + cur.execute('insert into test values(1)') + con.commit() + with pytest.raises(StopIteration): + next(cur) + +def test_cursor_after_close(con): + cur = con.execute('select 1') + cur.close() + con.close() + with pytest.raises(_sqlite3.ProgrammingError): + cur.close() + # raises ProgrammingError because should check closed before check args + with pytest.raises(_sqlite3.ProgrammingError): + cur.execute(1,2,3,4,5) + with pytest.raises(_sqlite3.ProgrammingError): + cur.executemany(1,2,3,4,5) + + at pypy_only +def test_connection_del(tmpdir): + """For issue1325.""" + import os + import gc + resource = pytest.importorskip('resource') + + limit = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + fds = 0 + while True: + fds += 1 + resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) + try: + for p in os.pipe(): os.close(p) + except OSError: + assert fds < 100 + else: + break + + def open_many(cleanup): + con = [] + for i in range(3): + con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) + if cleanup: + con[i] = None + gc.collect(); gc.collect() + + with pytest.raises(_sqlite3.OperationalError): + open_many(False) + sys.exc_clear() + gc.collect(); gc.collect() + open_many(True) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, limit) + +def test_on_conflict_rollback_executemany(con): + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) < (3, 2, 2): + pytest.skip("requires sqlite3 version >= 3.2.2") + con.execute("create table foo(x, unique(x) on conflict rollback)") + con.execute("insert into foo(x) values (1)") + try: + con.executemany("insert into foo(x) values (?)", [[1]]) + except _sqlite3.DatabaseError: + pass + con.execute("insert into foo(x) values (2)") + try: + con.commit() + except _sqlite3.OperationalError: + pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") + +def test_statement_arg_checking(con): + with pytest.raises(_sqlite3.Warning) as e: + con(123) + assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + with pytest.raises(ValueError) as e: + con.execute(123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executemany(123, 123) + assert str(e.value) == 'operation parameter must be str or unicode' + with pytest.raises(ValueError) as e: + con.executescript(123) + assert str(e.value) == 'script argument must be unicode or string.' + +def test_statement_param_checking(con): + con.execute('create table foo(x)') + con.execute('insert into foo(x) values (?)', [2]) + con.execute('insert into foo(x) values (?)', (2,)) + class seq(object): + def __len__(self): + return 1 + def __getitem__(self, key): + return 2 + con.execute('insert into foo(x) values (?)', seq()) + del seq.__len__ + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', seq()) + with pytest.raises(_sqlite3.ProgrammingError): + con.execute('insert into foo(x) values (?)', {2:2}) + with pytest.raises(ValueError) as e: + con.execute('insert into foo(x) values (?)', 2) + assert str(e.value) == 'parameters are of unsupported type' + +def test_explicit_begin(con): + con.execute('BEGIN') + con.execute('BEGIN ') + con.execute('BEGIN') + con.commit() + con.execute('BEGIN') + con.commit() + +def test_row_factory_use(con): + con.row_factory = 42 + con.execute('select 1') + +def test_returning_blob_must_own_memory(con): + import gc + con.create_function("returnblob", 0, lambda: buffer("blob")) + cur = con.execute("select returnblob()") + val = cur.fetchone()[0] + for i in range(5): + gc.collect() + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'l', 'o', 'b') + # in theory 'val' should be a read-write buffer + # but it's not right now + if not hasattr(_sqlite3, '_ffi'): + val[1] = 'X' + got = (val[0], val[1], val[2], val[3]) + assert got == ('b', 'X', 'o', 'b') + +def test_description_after_fetchall(con): + cur = con.cursor() + assert cur.description is None + cur.execute("select 42").fetchall() + assert cur.description is not None + +def test_executemany_lastrowid(con): + cur = con.cursor() + cur.execute("create table test(a)") + cur.executemany("insert into test values (?)", [[1], [2], [3]]) + assert cur.lastrowid is None + # issue 2682 + cur.execute('''insert + into test + values (?) + ''', (1, )) + assert cur.lastrowid is not None + cur.execute('''insert\t into test values (?) ''', (1, )) + assert cur.lastrowid is not None + +def test_authorizer_bad_value(con): + def authorizer_cb(action, arg1, arg2, dbname, source): + return 42 + con.set_authorizer(authorizer_cb) + with pytest.raises(_sqlite3.OperationalError) as e: + con.execute('select 123') + major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] + if (int(major), int(minor), int(micro)) >= (3, 6, 14): + assert str(e.value) == 'authorizer malfunction' + else: + assert str(e.value) == \ + ("illegal return value (1) from the authorization function - " + "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") + +def test_issue1573(con): + cur = con.cursor() + cur.execute(u'SELECT 1 as méil') + assert cur.description[0][0] == u"méil".encode('utf-8') + +def test_adapter_exception(con): + def cast(obj): + raise ZeroDivisionError + + _sqlite3.register_adapter(int, cast) + try: cur = con.cursor() - with pytest.raises(StopIteration): - next(cur) + cur.execute("select ?", (4,)) + val = cur.fetchone()[0] + # Adapter error is ignored, and parameter is passed as is. + assert val == 4 + assert type(val) is int + finally: + del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - cur.execute('select 1') - next(cur) - with pytest.raises(StopIteration): - next(cur) +def test_null_character(con): + if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): + pytest.skip("_sqlite3 too old") + with raises(ValueError) as excinfo: + con("\0select 1") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + con("select 1\0") + assert str(excinfo.value) == "the query contains a null character" + cur = con.cursor() + with raises(ValueError) as excinfo: + cur.execute("\0select 2") + assert str(excinfo.value) == "the query contains a null character" + with raises(ValueError) as excinfo: + cur.execute("select 2\0") + assert str(excinfo.value) == "the query contains a null character" - cur.execute('select 1') - con.commit() - next(cur) - with pytest.raises(StopIteration): - next(cur) - - with pytest.raises(_sqlite3.ProgrammingError): - cur.executemany('select 1', []) - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('create table test(ing)') - with pytest.raises(StopIteration): - next(cur) - - cur.execute('select 1') - cur.execute('insert into test values(1)') - con.commit() - with pytest.raises(StopIteration): - next(cur) - - def test_cursor_after_close(self, con): - cur = con.execute('select 1') - cur.close() - con.close() - pytest.raises(_sqlite3.ProgrammingError, "cur.close()") - # raises ProgrammingError because should check closed before check args - pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)") - pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)") - - @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')") - def test_connection_del(self, tmpdir): - """For issue1325.""" - import os - import gc - try: - import resource - except ImportError: - pytest.skip("needs resource module") - - limit = resource.getrlimit(resource.RLIMIT_NOFILE) - try: - fds = 0 - while True: - fds += 1 - resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) - try: - for p in os.pipe(): os.close(p) - except OSError: - assert fds < 100 - else: - break - - def open_many(cleanup): - con = [] - for i in range(3): - con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) - if cleanup: - con[i] = None - gc.collect(); gc.collect() - - pytest.raises(_sqlite3.OperationalError, open_many, False) - gc.collect(); gc.collect() - open_many(True) - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, limit) - - def test_on_conflict_rollback_executemany(self, con): - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) < (3, 2, 2): - pytest.skip("requires sqlite3 version >= 3.2.2") - con.execute("create table foo(x, unique(x) on conflict rollback)") - con.execute("insert into foo(x) values (1)") - try: - con.executemany("insert into foo(x) values (?)", [[1]]) - except _sqlite3.DatabaseError: - pass - con.execute("insert into foo(x) values (2)") - try: - con.commit() - except _sqlite3.OperationalError: - pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK") - - def test_statement_arg_checking(self, con): - with pytest.raises(_sqlite3.Warning) as e: - con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' - with pytest.raises(ValueError) as e: - con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' - with pytest.raises(ValueError) as e: - con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' - - def test_statement_param_checking(self, con): - con.execute('create table foo(x)') - con.execute('insert into foo(x) values (?)', [2]) - con.execute('insert into foo(x) values (?)', (2,)) - class seq(object): - def __len__(self): - return 1 - def __getitem__(self, key): - return 2 - con.execute('insert into foo(x) values (?)', seq()) - del seq.__len__ - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', seq()) - with pytest.raises(_sqlite3.ProgrammingError): - con.execute('insert into foo(x) values (?)', {2:2}) - with pytest.raises(ValueError) as e: - con.execute('insert into foo(x) values (?)', 2) - assert str(e.value) == 'parameters are of unsupported type' - - def test_explicit_begin(self, con): - con.execute('BEGIN') - con.execute('BEGIN ') - con.execute('BEGIN') - con.commit() - con.execute('BEGIN') - con.commit() - - def test_row_factory_use(self, con): - con.row_factory = 42 - con.execute('select 1') - - def test_returning_blob_must_own_memory(self, con): - import gc - con.create_function("returnblob", 0, lambda: buffer("blob")) - cur = con.execute("select returnblob()") - val = cur.fetchone()[0] - for i in range(5): - gc.collect() - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'l', 'o', 'b') - # in theory 'val' should be a read-write buffer - # but it's not right now - if not hasattr(_sqlite3, '_ffi'): - val[1] = 'X' - got = (val[0], val[1], val[2], val[3]) - assert got == ('b', 'X', 'o', 'b') - - def test_description_after_fetchall(self, con): - cur = con.cursor() - assert cur.description is None - cur.execute("select 42").fetchall() - assert cur.description is not None - - def test_executemany_lastrowid(self, con): - cur = con.cursor() - cur.execute("create table test(a)") - cur.executemany("insert into test values (?)", [[1], [2], [3]]) - assert cur.lastrowid is None - # issue 2682 - cur.execute('''insert - into test - values (?) - ''', (1, )) - assert cur.lastrowid is not None - cur.execute('''insert\t into test values (?) ''', (1, )) - assert cur.lastrowid is not None - - def test_authorizer_bad_value(self, con): - def authorizer_cb(action, arg1, arg2, dbname, source): - return 42 - con.set_authorizer(authorizer_cb) - with pytest.raises(_sqlite3.OperationalError) as e: - con.execute('select 123') - major, minor, micro = _sqlite3.sqlite_version.split('.')[:3] - if (int(major), int(minor), int(micro)) >= (3, 6, 14): - assert str(e.value) == 'authorizer malfunction' - else: - assert str(e.value) == \ - ("illegal return value (1) from the authorization function - " - "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY") - - def test_issue1573(self, con): - cur = con.cursor() - cur.execute(u'SELECT 1 as méil') - assert cur.description[0][0] == u"méil".encode('utf-8') - - def test_adapter_exception(self, con): - def cast(obj): - raise ZeroDivisionError - - _sqlite3.register_adapter(int, cast) - try: - cur = con.cursor() - cur.execute("select ?", (4,)) - val = cur.fetchone()[0] - # Adapter error is ignored, and parameter is passed as is. - assert val == 4 - assert type(val) is int - finally: - del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)] - - def test_null_character(self, con): - if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): - pytest.skip("_sqlite3 too old") - exc = raises(ValueError, con, "\0select 1") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, con, "select 1\0") - assert str(exc.value) == "the query contains a null character" - cur = con.cursor() - exc = raises(ValueError, cur.execute, "\0select 2") - assert str(exc.value) == "the query contains a null character" - exc = raises(ValueError, cur.execute, "select 2\0") - assert str(exc.value) == "the query contains a null character" - - def test_close_in_del_ordering(self): - import gc - class SQLiteBackend(object): - success = False - def __init__(self): - self.connection = _sqlite3.connect(":memory:") - def close(self): - self.connection.close() - def __del__(self): - self.close() - SQLiteBackend.success = True - def create_db_if_needed(self): - conn = self.connection - cursor = conn.cursor() - cursor.execute(""" - create table if not exists nameoftable(value text) - """) - cursor.close() - conn.commit() - SQLiteBackend().create_db_if_needed() - gc.collect() - gc.collect() - assert SQLiteBackend.success - - -class TestSQLiteHost(BaseTestSQLite): - def setup_class(cls): - global _sqlite3 - import _sqlite3 - - -class TestSQLitePyPy(BaseTestSQLite): - def setup_class(cls): - if sys.version_info < (2, 7): - pytest.skip("_sqlite3 requires Python 2.7") - - try: - from lib_pypy import _sqlite3_cffi - except ImportError: - # On CPython, "pip install cffi". On old PyPy's, no chance - pytest.skip("install cffi and run lib_pypy/_sqlite3_build.py " - "manually first") - - global _sqlite3 - from lib_pypy import _sqlite3 +def test_close_in_del_ordering(): + import gc + class SQLiteBackend(object): + success = False + def __init__(self): + self.connection = _sqlite3.connect(":memory:") + def close(self): + self.connection.close() + def __del__(self): + self.close() + SQLiteBackend.success = True + def create_db_if_needed(self): + conn = self.connection + cursor = conn.cursor() + cursor.execute(""" + create table if not exists nameoftable(value text) + """) + cursor.close() + conn.commit() + SQLiteBackend().create_db_if_needed() + gc.collect() + gc.collect() + assert SQLiteBackend.success From pypy.commits at gmail.com Thu Nov 29 00:58:03 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Nov 2018 21:58:03 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Move test_sqlite3.py to extra_tests/ Message-ID: <5bff7feb.1c69fb81.ebe95.a22a@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95380:8e5be49bf7c4 Date: 2018-11-29 05:29 +0000 http://bitbucket.org/pypy/pypy/changeset/8e5be49bf7c4/ Log: Move test_sqlite3.py to extra_tests/ diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/extra_tests/test_sqlite3.py rename from pypy/module/test_lib_pypy/test_sqlite3.py rename to extra_tests/test_sqlite3.py --- a/pypy/module/test_lib_pypy/test_sqlite3.py +++ b/extra_tests/test_sqlite3.py @@ -286,17 +286,17 @@ def test_null_character(con): if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9): pytest.skip("_sqlite3 too old") - with raises(ValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: con("\0select 1") assert str(excinfo.value) == "the query contains a null character" - with raises(ValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: con("select 1\0") assert str(excinfo.value) == "the query contains a null character" cur = con.cursor() - with raises(ValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: cur.execute("\0select 2") assert str(excinfo.value) == "the query contains a null character" - with raises(ValueError) as excinfo: + with pytest.raises(ValueError) as excinfo: cur.execute("select 2\0") assert str(excinfo.value) == "the query contains a null character" From pypy.commits at gmail.com Thu Nov 29 00:58:05 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Nov 2018 21:58:05 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: rm workaround for https://github.com/pytest-dev/pytest/issues/1965 Message-ID: <5bff7fed.1c69fb81.68b19.2cc8@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95381:ad49d07c6e0b Date: 2018-11-29 05:49 +0000 http://bitbucket.org/pypy/pypy/changeset/ad49d07c6e0b/ Log: rm workaround for https://github.com/pytest-dev/pytest/issues/1965 diff --git a/extra_tests/test_sqlite3.py b/extra_tests/test_sqlite3.py --- a/extra_tests/test_sqlite3.py +++ b/extra_tests/test_sqlite3.py @@ -116,7 +116,6 @@ with pytest.raises(_sqlite3.ProgrammingError): cur.executemany(1,2,3,4,5) - at pypy_only def test_connection_del(tmpdir): """For issue1325.""" import os @@ -146,7 +145,6 @@ with pytest.raises(_sqlite3.OperationalError): open_many(False) - sys.exc_clear() gc.collect(); gc.collect() open_many(True) finally: From pypy.commits at gmail.com Thu Nov 29 00:58:06 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 28 Nov 2018 21:58:06 -0800 (PST) Subject: [pypy-commit] pypy cleanup-test_lib_pypy: Make tests more py3-friendly Message-ID: <5bff7fee.1c69fb81.e89c4.4463@mx.google.com> Author: Ronan Lamy Branch: cleanup-test_lib_pypy Changeset: r95382:d2c59938bd7b Date: 2018-11-29 05:57 +0000 http://bitbucket.org/pypy/pypy/changeset/d2c59938bd7b/ Log: Make tests more py3-friendly diff --git a/extra_tests/test_sqlite3.py b/extra_tests/test_sqlite3.py --- a/extra_tests/test_sqlite3.py +++ b/extra_tests/test_sqlite3.py @@ -51,7 +51,7 @@ con = Connection(":memory:") with pytest.raises(_sqlite3.ProgrammingError) as excinfo: con.cursor() - assert '__init__' in excinfo.value.message + assert '__init__' in str(excinfo.value) def test_cursor_check_init(con): @@ -62,7 +62,8 @@ cur = Cursor(con) with pytest.raises(_sqlite3.ProgrammingError) as excinfo: cur.execute('select 1') - assert '__init__' in excinfo.value.message + assert '__init__' in str(excinfo.value) + def test_connection_after_close(con): with pytest.raises(TypeError): @@ -169,16 +170,16 @@ def test_statement_arg_checking(con): with pytest.raises(_sqlite3.Warning) as e: con(123) - assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.' + assert str(e.value).startswith('SQL is of wrong type. Must be string') with pytest.raises(ValueError) as e: con.execute(123) - assert str(e.value) == 'operation parameter must be str or unicode' + assert str(e.value).startswith('operation parameter must be str') with pytest.raises(ValueError) as e: con.executemany(123, 123) - assert str(e.value) == 'operation parameter must be str or unicode' + assert str(e.value).startswith('operation parameter must be str') with pytest.raises(ValueError) as e: con.executescript(123) - assert str(e.value) == 'script argument must be unicode or string.' + assert str(e.value).startswith('script argument must be unicode') def test_statement_param_checking(con): con.execute('create table foo(x)') From pypy.commits at gmail.com Thu Nov 29 06:30:06 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 29 Nov 2018 03:30:06 -0800 (PST) Subject: [pypy-commit] pypy default: A failing test based on #2904 Message-ID: <5bffcdbe.1c69fb81.c46ec.9384@mx.google.com> Author: Armin Rigo Branch: Changeset: r95383:fd8dbffff070 Date: 2018-11-29 13:29 +0200 http://bitbucket.org/pypy/pypy/changeset/fd8dbffff070/ Log: A failing test based on #2904 diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,3 +4702,38 @@ res = self.meta_interp(f, [10]) assert res == f(10) + def test_issue2904(self): + driver = JitDriver(greens=['codepos'], + reds=['iterations', 'total', 'c', 'height', 'h']) + + class IntVal: + _immutable_fields_ = ['intval'] + def __init__(self, value): + self.intval = value + + def f(height, iterations): + height = IntVal(height) + c = IntVal(0) + h = IntVal(0) + total = IntVal(0) + codepos = 1 + + while True: + driver.jit_merge_point(codepos=codepos, iterations=iterations, + total=total, c=c, height=height, h=h) + if codepos == 1: + if c.intval >= iterations: + return total.intval + else: + h = height + codepos = 2 + else: + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) + codepos = 1 + + res = self.meta_interp(f, [2, 200]) + assert res == f(2, 200) From pypy.commits at gmail.com Thu Nov 29 06:35:55 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 29 Nov 2018 03:35:55 -0800 (PST) Subject: [pypy-commit] pypy default: Simplify a bit Message-ID: <5bffcf1b.1c69fb81.5655.8060@mx.google.com> Author: Armin Rigo Branch: Changeset: r95384:ddf5bd822a68 Date: 2018-11-29 13:35 +0200 http://bitbucket.org/pypy/pypy/changeset/ddf5bd822a68/ Log: Simplify a bit diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4703,7 +4703,7 @@ assert res == f(10) def test_issue2904(self): - driver = JitDriver(greens=['codepos'], + driver = JitDriver(greens = [], reds=['iterations', 'total', 'c', 'height', 'h']) class IntVal: @@ -4714,26 +4714,20 @@ def f(height, iterations): height = IntVal(height) c = IntVal(0) - h = IntVal(0) + h = height total = IntVal(0) - codepos = 1 while True: - driver.jit_merge_point(codepos=codepos, iterations=iterations, + driver.jit_merge_point(iterations=iterations, total=total, c=c, height=height, h=h) - if codepos == 1: + if h.intval > 0: + h = IntVal(h.intval - 1) + total = IntVal(total.intval + 1) + else: + c = IntVal(c.intval + 1) if c.intval >= iterations: return total.intval - else: - h = height - codepos = 2 - else: - if h.intval > 0: - h = IntVal(h.intval - 1) - total = IntVal(total.intval + 1) - else: - c = IntVal(c.intval + 1) - codepos = 1 + h = height res = self.meta_interp(f, [2, 200]) assert res == f(2, 200) From pypy.commits at gmail.com Thu Nov 29 08:26:00 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 29 Nov 2018 05:26:00 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: one test uncovered two fixes Message-ID: <5bffe8e8.1c69fb81.91fc9.66e0@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95385:96d5bc8106a8 Date: 2018-11-29 05:24 -0800 http://bitbucket.org/pypy/pypy/changeset/96d5bc8106a8/ Log: one test uncovered two fixes diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -749,7 +749,7 @@ state = space.fromcache(CodecState) # call the fast version for checking try: - lgt = rutf8.check_utf8(string, allow_surrogates=True) + lgt = rutf8.check_utf8(string, allow_surrogates=False) except rutf8.CheckError: res, lgt, pos = unicodehelper.str_decode_utf8(string, errors, final, state.decode_error_handler) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -625,6 +625,8 @@ assert '[\uDC80]'.encode('utf-8', 'namereplace') == b'[\\udc80]' def test_surrogateescape(self): + uni = b"\xed\xb0\x80".decode("utf-8", "surrogateescape") + assert uni == "\udced\udcb0\udc80" assert "\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape") == b"\xe4\xeb\xef\xf6\xfc" assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b' diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -198,11 +198,13 @@ if encoding is None: encoding = 'utf8' if encoding == 'utf8' or encoding == 'utf-8': + # fast path - do not call into app-level codecs.py from pypy.module._codecs.interp_codecs import CodecState state = space.fromcache(CodecState) eh = state.decode_error_handler s = space.charbuf_w(self) ret, lgt, pos = str_decode_utf8(s, errors, True, eh) + return space.newtext(ret, lgt) return decode_object(space, self, encoding, errors) @unwrap_spec(tabsize=int) From pypy.commits at gmail.com Thu Nov 29 12:21:22 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 29 Nov 2018 09:21:22 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: test, fix surrogates in _sre encoding Message-ID: <5c002012.1c69fb81.61b6c.5510@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95386:3a259b9aeedd Date: 2018-11-29 08:02 -0800 http://bitbucket.org/pypy/pypy/changeset/3a259b9aeedd/ Log: test, fix surrogates in _sre encoding diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -43,7 +43,8 @@ return space.newbytes(ctx._string[start:end]) elif isinstance(ctx, rsre_core.UnicodeMatchContext): uni = ctx._unicodestr[start:end] - uni_utf8 = unicode_encode_utf_8(uni, len(uni), 'strict') + uni_utf8 = unicode_encode_utf_8(uni, len(uni), 'strict', + allow_surrogates=True) return space.newtext(uni_utf8, len(uni)) else: # unreachable diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -116,6 +116,8 @@ assert ['', 'a', 'l', 'a', 'lla'] == re.split("b(a)", "balballa") assert ['', 'a', None, 'l', 'u', None, 'lla'] == ( re.split("b([ua]|(s))", "balbulla")) + assert ['Hello \udce2\udc9c\udc93', ''] == re.split(r'\r\n|\r|\n', + 'Hello \udce2\udc9c\udc93\n') def test_weakref(self): import re, _weakref From pypy.commits at gmail.com Thu Nov 29 12:21:25 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 29 Nov 2018 09:21:25 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8-py3: replace py.skip with py.mark.skip to unskip module, fix tests Message-ID: <5c002015.1c69fb81.6c465.56b3@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95387:24265ff12831 Date: 2018-11-29 09:20 -0800 http://bitbucket.org/pypy/pypy/changeset/24265ff12831/ Log: replace py.skip with py.mark.skip to unskip module, fix tests diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -19,9 +19,11 @@ raise Hit raise AttributeError(name) +def fake_eh(errors, encoding, msg, u, startingpos, endingpos): + raise Hit() def decode_utf8(u): - return str_decode_utf8(u, "strict", True, None) + return str_decode_utf8(u, "strict", True, fake_eh) def test_encode_utf8(): space = FakeSpace() @@ -52,22 +54,22 @@ def test_decode_utf8(): assert decode_utf8("abc") == ("abc", 3, 3) - assert decode_utf8("\xe1\x88\xb4") == ("\xe1\x88\xb4", 3, 1) - assert decode_utf8("\xed\xa0\x80") == ("\xed\xa0\x80", 3, 1) + assert decode_utf8("\xe1\x88\xb4") == ("\xe1\x88\xb4", 1, 3) py.test.raises(Hit, decode_utf8, "\xed\xa0\x80") py.test.raises(Hit, decode_utf8, "\xed\xb0\x80") py.test.raises(Hit, decode_utf8, "\xed\xa0\x80\xed\xb0\x80") got = decode_utf8("\xf0\x90\x80\x80") if sys.maxunicode > 65535: - assert map(ord, got) == [0x10000] + assert got == ("\xf0\x90\x80\x80", 1, 4) else: + # never reached assert map(ord, got) == [55296, 56320] def test_utf8_encode_ascii(): assert utf8_encode_ascii("abc", "??", "??") == "abc" def eh(errors, encoding, reason, p, start, end): lst.append((errors, encoding, p, start, end)) - return "", end + return "", end, 'b' lst = [] input = u"\u1234".encode("utf8") assert utf8_encode_ascii(input, "??", eh) == "" @@ -78,7 +80,7 @@ assert lst == [("??", "ascii", input, 0, 2), ("??", "ascii", input, 5, 7)] - at pytest.skip("rework this test for utf8") + at pytest.mark.skip("rework this test for utf8") def test_decode_utf8_allow_surrogates(): sp = FakeSpace() assert decode_utf8(sp, "\xed\xa0\x80", allow_surrogates=True) == u"\ud800" @@ -88,7 +90,7 @@ got = decode_utf8(sp, "\xf0\x90\x80\x80", allow_surrogates=True) assert map(ord, got) == [0x10000] - at pytest.skip("rework this test for utf8") + at pytest.mark.skip("rework this test for utf8") def test_decode_utf8sp(): space = FakeSpace() assert decode_utf8sp(space, "\xed\xa0\x80") == u"\ud800" @@ -98,7 +100,7 @@ got = decode_utf8sp(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] - at pytest.skip("test has non-valid errorhandler") + at pytest.mark.skip("test has non-valid errorhandler") @pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) def test_utf32_surrogates(unich): assert (utf8_encode_utf_32_be(unich.encode('utf-8'), None) == @@ -132,23 +134,23 @@ @given(strategies.text()) def test_utf8_encode_ascii_2(u): def eh(errors, encoding, reason, p, start, end): - return "?" * (end - start), end + return "?" * (end - start), end, 'b' assert utf8_encode_ascii(u.encode("utf8"), "replace", eh) == u.encode("ascii", "replace") def test_str_decode_ascii(): assert str_decode_ascii("abc", "??", True, "??") == ("abc", 3, 3) def eh(errors, encoding, reason, p, start, end): lst.append((errors, encoding, p, start, end)) - return u"\u1234\u5678".encode("utf8"), end + return u"\u1234\u5678".encode("utf8"), end, 'u' lst = [] input = "\xe8" exp = u"\u1234\u5678".encode("utf8") - assert str_decode_ascii(input, "??", True, eh) == (exp, 1, 2) + assert str_decode_ascii(input, "??", True, eh) == (exp, 2, 2) assert lst == [("??", "ascii", input, 0, 1)] lst = [] input = "\xe8\xe9abc\xea\xeb" assert str_decode_ascii(input, "??", True, eh) == ( - exp + exp + "abc" + exp + exp, 7, 11) + exp + exp + "abc" + exp + exp, 11, 11) assert lst == [("??", "ascii", input, 0, 1), ("??", "ascii", input, 1, 2), ("??", "ascii", input, 5, 6), From pypy.commits at gmail.com Thu Nov 29 13:24:08 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 29 Nov 2018 10:24:08 -0800 (PST) Subject: [pypy-commit] cffi default: Merged in rlamy/cffi (pull request #94) Message-ID: <5c002ec8.1c69fb81.5af47.f580@mx.google.com> Author: Armin Rigo Branch: Changeset: r3174:448af882f47e Date: 2018-11-29 18:24 +0000 http://bitbucket.org/cffi/cffi/changeset/448af882f47e/ Log: Merged in rlamy/cffi (pull request #94) Move test-only function from cffi/ to testing/ diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -1540,27 +1540,3 @@ else: return None, updated -def _verify(ffi, module_name, preamble, *args, **kwds): - # FOR TESTS ONLY - from testing.udir import udir - import imp - assert module_name not in sys.modules, "module name conflict: %r" % ( - module_name,) - kwds.setdefault('tmpdir', str(udir)) - outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) - module = imp.load_dynamic(module_name, outputfilename) - # - # hack hack hack: copy all *bound methods* from module.ffi back to the - # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). - for name in dir(module.ffi): - if not name.startswith('_'): - attr = getattr(module.ffi, name) - if attr is not getattr(ffi, name, object()): - setattr(ffi, name, attr) - def typeof_disabled(*args, **kwds): - raise NotImplementedError - ffi._typeof = typeof_disabled - for name in dir(ffi): - if not name.startswith('_') and not hasattr(module.ffi, name): - setattr(ffi, name, NotImplemented) - return module.lib diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -4,7 +4,7 @@ from cffi import recompiler from testing.udir import udir from testing.support import u, long -from testing.support import FdWriteCapture, StdErrCapture +from testing.support import FdWriteCapture, StdErrCapture, _verify try: import importlib @@ -35,7 +35,7 @@ # add '-Werror' to the existing 'extra_compile_args' flags kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + ['-Werror']) - return recompiler._verify(ffi, module_name, source, *args, **kwds) + return _verify(ffi, module_name, source, *args, **kwds) def test_set_source_no_slashes(): ffi = FFI() diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py --- a/testing/cffi1/test_verify1.py +++ b/testing/cffi1/test_verify1.py @@ -3,6 +3,7 @@ from cffi import CDefError from cffi import recompiler from testing.support import * +from testing.support import _verify import _cffi_backend lib_m = ['m'] @@ -37,9 +38,8 @@ except AttributeError: pass self.set_source(module_name, preamble) - return recompiler._verify(self, module_name, preamble, *args, - extra_compile_args=self._extra_compile_args, - **kwds) + return _verify(self, module_name, preamble, *args, + extra_compile_args=self._extra_compile_args, **kwds) class FFI_warnings_not_error(FFI): _extra_compile_args = [] diff --git a/testing/support.py b/testing/support.py --- a/testing/support.py +++ b/testing/support.py @@ -61,3 +61,28 @@ def getvalue(self): return self._value + +def _verify(ffi, module_name, preamble, *args, **kwds): + import imp + from cffi.recompiler import recompile + from .udir import udir + assert module_name not in sys.modules, "module name conflict: %r" % ( + module_name,) + kwds.setdefault('tmpdir', str(udir)) + outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) + module = imp.load_dynamic(module_name, outputfilename) + # + # hack hack hack: copy all *bound methods* from module.ffi back to the + # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). + for name in dir(module.ffi): + if not name.startswith('_'): + attr = getattr(module.ffi, name) + if attr is not getattr(ffi, name, object()): + setattr(ffi, name, attr) + def typeof_disabled(*args, **kwds): + raise NotImplementedError + ffi._typeof = typeof_disabled + for name in dir(ffi): + if not name.startswith('_') and not hasattr(module.ffi, name): + setattr(ffi, name, NotImplemented) + return module.lib From pypy.commits at gmail.com Fri Nov 30 03:11:28 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 30 Nov 2018 00:11:28 -0800 (PST) Subject: [pypy-commit] pypy default: Skip the test as showing an unrelated and maybe not fatal problem. Message-ID: <5c00f0b0.1c69fb81.7a6d.8139@mx.google.com> Author: Armin Rigo Branch: Changeset: r95388:6ee30aff8e5c Date: 2018-11-30 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/6ee30aff8e5c/ Log: Skip the test as showing an unrelated and maybe not fatal problem. Add a new test that really shows the problem, unless I'm confused again. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9499,5 +9499,116 @@ """ self.optimize_loop(ops, expected) + def test_issue2904(self): + py.test.skip("XXX issue 2904") + ops = """ + [p0, p1, p2, p3, p4, i5, p6, i7, p8, p9, p10, p11, p12, p13] + debug_merge_point(0, 0, ' #36 LOAD_FAST') + guard_value(i5, 4) [] + guard_isnull(p3) [] + guard_nonnull(p11) [] + debug_merge_point(0, 0, ' #39 LOAD_CONST') + guard_value(p2, ConstPtr(myptr)) [] + debug_merge_point(0, 0, ' #42 COMPARE_OP') + guard_class(p11, ConstClass(intobj_immut_vtable)) [] + i17 = getfield_gc_i(p11, descr=immut_intval) + i19 = int_gt(i17, 0) + guard_false(i19) [] + debug_merge_point(0, 0, ' #45 POP_JUMP_IF_FALSE') + debug_merge_point(0, 0, ' #71 POP_BLOCK') + p20 = getfield_gc_r(p4, descr=nextdescr3) + i21 = getfield_gc_i(p4, descr=valuedescr3) + guard_value(i21, 4) [] + debug_merge_point(0, 0, ' #72 LOAD_FAST') + guard_nonnull(p10) [] + debug_merge_point(0, 0, ' #75 LOAD_CONST') + debug_merge_point(0, 0, ' #78 INPLACE_ADD') + guard_class(p10, ConstClass(intobj_immut_vtable)) [] + i24 = getfield_gc_i(p10, descr=immut_intval) + i26 = int_add_ovf(i24, 1) + guard_no_overflow() [] + p27 = new_with_vtable(descr=immut_descr) + setfield_gc(p27, i26, descr=immut_intval) + debug_merge_point(0, 0, ' #79 STORE_FAST') + debug_merge_point(0, 0, ' #82 JUMP_ABSOLUTE') + quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr) + guard_not_invalidated() [] + i30 = getfield_raw_i(140031323711392, descr=adescr) + i32 = int_lt(i30, 0) + guard_false(i32) [] + debug_merge_point(0, 0, ' #15 LOAD_FAST') + p34 = same_as_r(ConstPtr(tupleaddr)) + p36 = same_as_r(ConstPtr(nullptr)) + i38 = same_as_i(4) + i40 = same_as_i(15) + p42 = same_as_r(ConstPtr(nullptr)) + p44 = same_as_r(ConstPtr(nullptr)) + guard_future_condition() [] + guard_value(i38, 4) [] + guard_isnull(p36) [] + guard_nonnull(p27) [] + debug_merge_point(0, 0, ' #18 LOAD_CONST') + guard_value(p34, ConstPtr(tupleaddr)) [] + debug_merge_point(0, 0, ' #21 COMPARE_OP') + guard_class(p27, ConstClass(intobj_immut_vtable)) [] + i48 = getfield_gc_i(p27, descr=immut_intval) + i50 = int_lt(i48, 60000) + guard_true(i50) [] + debug_merge_point(0, 0, ' #24 POP_JUMP_IF_FALSE') + debug_merge_point(0, 0, ' #27 LOAD_FAST') + guard_nonnull(p8) [] + debug_merge_point(0, 0, ' #30 STORE_FAST') + debug_merge_point(0, 0, ' #33 SETUP_LOOP') + p51 = new_with_vtable(descr=nodesize) + setfield_gc(p51, 72, descr=valuedescr3) + setfield_gc(p51, 4, descr=immut_intval) + setfield_gc(p51, p20, descr=nextdescr3) + debug_merge_point(0, 0, ' #36 LOAD_FAST') + debug_merge_point(0, 0, ' #39 LOAD_CONST') + debug_merge_point(0, 0, ' #42 COMPARE_OP') + guard_class(p8, ConstClass(intobj_immut_vtable)) [] + i55 = getfield_gc_i(p8, descr=immut_intval) + i57 = int_gt(i55, 0) + guard_true(i57) [] + debug_merge_point(0, 0, ' #45 POP_JUMP_IF_FALSE') + debug_merge_point(0, 0, ' #48 LOAD_FAST') + debug_merge_point(0, 0, ' #51 LOAD_CONST') + debug_merge_point(0, 0, ' #54 INPLACE_SUBTRACT') + i59 = int_sub_ovf(i55, 1) + guard_no_overflow() [] + p60 = new_with_vtable(descr=immut_descr) + setfield_gc(p60, i59, descr=immut_intval) + debug_merge_point(0, 0, ' #55 STORE_FAST') + debug_merge_point(0, 0, ' #58 LOAD_FAST') + guard_nonnull(p9) [] + debug_merge_point(0, 0, ' #61 LOAD_CONST') + debug_merge_point(0, 0, ' #64 INPLACE_ADD') + guard_class(p9, ConstClass(intobj_immut_vtable)) [] + i62 = getfield_gc_i(p9, descr=immut_intval) + i64 = int_add_ovf(i62, 1) + guard_no_overflow() [] + p65 = new_with_vtable(descr=immut_descr) + setfield_gc(p65, i64, descr=immut_intval) + debug_merge_point(0, 0, ' #65 STORE_FAST') + debug_merge_point(0, 0, ' #68 JUMP_ABSOLUTE') + quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr) + guard_not_invalidated() [] + i68 = getfield_raw_i(140031323711392, descr=adescr) + i70 = int_lt(i68, 0) + guard_false(i70) [] + debug_merge_point(0, 0, ' #36 LOAD_FAST') + p72 = same_as_r(ConstPtr(myptr)) + p74 = same_as_r(ConstPtr(nullptr)) + i76 = same_as_i(4) + i78 = same_as_i(36) + p80 = same_as_r(ConstPtr(nullptr)) + p82 = same_as_r(ConstPtr(nullptr)) + guard_future_condition() [] + jump(p0, p1, p72, p74, p51, i76, p6, i78, p8, p65, p27, p60, p80, p82, descr=) + """ + # expected = a loop that does NOT end up passing the constant 0 in the final jump() + self.optimize_loop(ops, ops, + jump_values=[None] * 14) + class TestLLtype(OptimizeOptTest, LLtypeMixin): pass diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4702,7 +4702,8 @@ res = self.meta_interp(f, [10]) assert res == f(10) - def test_issue2904(self): + def test_cached_info_missing(self): + py.test.skip("XXX hitting a non-translated assert in optimizeopt/heap.py, but seems not to hurt the rest") driver = JitDriver(greens = [], reds=['iterations', 'total', 'c', 'height', 'h']) From pypy.commits at gmail.com Fri Nov 30 10:16:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 30 Nov 2018 07:16:48 -0800 (PST) Subject: [pypy-commit] pypy default: fix test Message-ID: <5c015460.1c69fb81.bfaba.ca74@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95389:a2d417aec82f Date: 2018-11-30 16:15 +0100 http://bitbucket.org/pypy/pypy/changeset/a2d417aec82f/ Log: fix test diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -521,11 +521,9 @@ assert path.check(file=0) def test_debug_start_stop_timestamp(self): - import sys - import time from rpython.rlib.rtimer import read_timestamp def entry_point(argv): - timestamp = int(argv[1]) + timestamp = bool(int(argv[1])) ts1 = debug_start("foo", timestamp=timestamp) ts2 = read_timestamp() ts3 = debug_stop("foo", timestamp=timestamp) From pypy.commits at gmail.com Fri Nov 30 10:21:02 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 30 Nov 2018 07:21:02 -0800 (PST) Subject: [pypy-commit] pypy default: merge expose-gc-time: Message-ID: <5c01555e.1c69fb81.06e8.5c00@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r95390:c8b8ee6787e9 Date: 2018-11-30 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/c8b8ee6787e9/ Log: merge expose-gc-time: expose the total time spent in the GC, also switch to using seconds in the GC hooks diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -51,3 +51,8 @@ .. branch: rlock-in-rpython Backport CPython fix for `thread.RLock` + + +.. branch: expose-gc-time + +Make GC hooks measure time in seconds (as opposed to an opaque unit). diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -368,7 +368,7 @@ def get_gchooks(self): from pypy.module.gc.hook import LowLevelGcHooks if self.space is None: - raise Exception("get_gchooks must be called afeter get_entry_point") + raise Exception("get_gchooks must be called after get_entry_point") return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -57,12 +57,14 @@ 'total_allocated_memory', 'jit_backend_allocated', 'peak_memory', 'peak_allocated_memory', 'total_arena_memory', 'total_rawmalloced_memory', 'nursery_size', - 'peak_arena_memory', 'peak_rawmalloced_memory'): + 'peak_arena_memory', 'peak_rawmalloced_memory', + ): setattr(self, item, self._format(getattr(self._s, item))) self.memory_used_sum = self._format(self._s.total_gc_memory + self._s.total_memory_pressure + self._s.jit_backend_used) self.memory_allocated_sum = self._format(self._s.total_allocated_memory + self._s.total_memory_pressure + self._s.jit_backend_allocated) + self.total_gc_time = self._s.total_gc_time def _format(self, v): if v < 1000000: @@ -92,6 +94,8 @@ raw assembler allocated: %s%s ----------------------------- Total: %s + + Total time spent in GC: %s """ % (self.total_gc_memory, self.peak_memory, self.total_arena_memory, self.total_rawmalloced_memory, @@ -106,7 +110,8 @@ self.nursery_size, self.jit_backend_allocated, extra, - self.memory_allocated_sum) + self.memory_allocated_sum, + self.total_gc_time / 1000.0) def get_stats(memory_pressure=False): diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -7,6 +7,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction +inf = float("inf") + class LowLevelGcHooks(GcHooks): """ These are the low-level hooks which are called directly from the GC. @@ -126,9 +128,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -136,9 +138,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -166,9 +168,9 @@ def reset(self): self.count = 0 - self.duration = r_longlong(0) - self.duration_min = r_longlong(longlongmax) - self.duration_max = r_longlong(0) + self.duration = 0.0 + self.duration_min = inf + self.duration_max = 0.0 def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -176,9 +178,9 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(r_longlong(-42)) - self.duration_min = NonConstant(r_longlong(-42)) - self.duration_max = NonConstant(r_longlong(-42)) + self.duration = NonConstant(-53.2) + self.duration_min = NonConstant(-53.2) + self.duration_max = NonConstant(-53.2) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -276,10 +278,14 @@ # just a shortcut to make the typedefs shorter -def wrap_many_ints(cls, names): +def wrap_many(cls, names): d = {} for name in names: - d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + if "duration" in name: + wrapfn = "newfloat" + else: + wrapfn = "newint" + d[name] = interp_attrproperty(name, cls=cls, wrapfn=wrapfn) return d @@ -303,7 +309,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", - **wrap_many_ints(W_GcMinorStats, ( + **wrap_many(W_GcMinorStats, ( "count", "duration", "duration_min", @@ -319,7 +325,7 @@ STATE_SWEEPING = incminimark.STATE_SWEEPING, STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), - **wrap_many_ints(W_GcCollectStepStats, ( + **wrap_many(W_GcCollectStepStats, ( "count", "duration", "duration_min", @@ -330,7 +336,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", - **wrap_many_ints(W_GcCollectStats, ( + **wrap_many(W_GcCollectStats, ( "count", "num_major_collects", "arenas_count_before", diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -189,6 +189,7 @@ self.peak_arena_memory = rgc.get_stats(rgc.PEAK_ARENA_MEMORY) self.peak_rawmalloced_memory = rgc.get_stats(rgc.PEAK_RAWMALLOCED_MEMORY) self.nursery_size = rgc.get_stats(rgc.NURSERY_SIZE) + self.total_gc_time = rgc.get_stats(rgc.TOTAL_GC_TIME) W_GcStats.typedef = TypeDef("GcStats", total_memory_pressure=interp_attrproperty("total_memory_pressure", @@ -215,6 +216,8 @@ cls=W_GcStats, wrapfn="newint"), nursery_size=interp_attrproperty("nursery_size", cls=W_GcStats, wrapfn="newint"), + total_gc_time=interp_attrproperty("total_gc_time", + cls=W_GcStats, wrapfn="newint"), ) @unwrap_spec(memory_pressure=bool) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,11 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(5, 0, 0) - gchooks.fire_gc_minor(7, 0, 0) - gchooks.fire_gc_collect_step(5, 0, 0) - gchooks.fire_gc_collect_step(15, 0, 0) - gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_minor(5.0, 0, 0) + gchooks.fire_gc_minor(7.0, 0, 0) + gchooks.fire_gc_collect_step(5.0, 0, 0) + gchooks.fire_gc_collect_step(15.0, 0, 0) + gchooks.fire_gc_collect_step(22.0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,6 +62,7 @@ # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys import os +import time from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -73,7 +74,6 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc -from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -192,6 +192,8 @@ # ____________________________________________________________ + + class IncrementalMiniMarkGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -374,6 +376,7 @@ self.raw_malloc_might_sweep = self.AddressStack() self.rawmalloced_total_size = r_uint(0) self.rawmalloced_peak_size = r_uint(0) + self.total_gc_time = 0.0 self.gc_state = STATE_SCANNING # @@ -1644,7 +1647,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # - start = read_timestamp() + start = time.time() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1843,7 +1846,8 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_minor( duration=duration, total_memory_used=total_memory_used, @@ -2249,7 +2253,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): - start = read_timestamp() + start = time.time() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2493,7 +2497,8 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - duration = read_timestamp() - start + duration = time.time() - start + self.total_gc_time += duration self.hooks.fire_gc_collect_step( duration=duration, oldstate=oldstate, @@ -3000,6 +3005,8 @@ self.ac.total_memory_used)) elif stats_no == rgc.NURSERY_SIZE: return intmask(self.nursery_size) + elif stats_no == rgc.TOTAL_GC_TIME: + return int(self.total_gc_time * 1000) return 0 diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -70,7 +70,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] - assert self.gc.hooks.durations[0] > 0 + assert self.gc.hooks.durations[0] > 0. self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -103,7 +103,7 @@ ] assert len(self.gc.hooks.durations) == 4 # 4 steps for d in self.gc.hooks.durations: - assert d > 0 + assert d > 0.0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -704,7 +704,7 @@ (TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE, PEAK_MEMORY, PEAK_ALLOCATED_MEMORY, TOTAL_ARENA_MEMORY, TOTAL_RAWMALLOCED_MEMORY, PEAK_ARENA_MEMORY, PEAK_RAWMALLOCED_MEMORY, - NURSERY_SIZE) = range(10) + NURSERY_SIZE, TOTAL_GC_TIME) = range(11) @not_rpython def get_stats(stat_no): diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1812,7 +1812,20 @@ res = self.run("ignore_finalizer") assert res == 1 # translated: x1 is removed from the list + def define_total_gc_time(cls): + def f(): + l = [] + for i in range(1000000): + l.append(str(i)) + l = [] + for i in range(10): + rgc.collect() + return rgc.get_stats(rgc.TOTAL_GC_TIME) + return f + def test_total_gc_time(self): + res = self.run("total_gc_time") + assert res > 0 # should take a few microseconds # ____________________________________________________________________ class TaggedPointersTest(object):